2026-02-22T19:43:45.957 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-22T19:43:45.977 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T19:43:46.062 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002 branch: wip-yuri6-testing-2026-02-22-1629-squid description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zstd supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-workunit-minimum} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '64002' kernel: branch: distro kdb: true sha1: distro ktype: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode name: yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: wip-yuri6-testing-2026-02-22-1629-squid ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.4: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.5: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.6: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluestore block size: 96636764160 bluestore compression algorithm: zstd bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-yuri6-testing-2026-02-22-1629-squid sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 9196 sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-yuri6-testing-2026-02-22-1629-squid suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 targets: trial012.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHr8AQoPmLBCDlwyLfkwSBORD6UrQX2i0NkRJcfXP05UjfcnISwAcRU/7PWmtbs6gaA2+U8uoxsT8lx4AB9jXYs= trial098.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFhjjShomKSln4JXn92gq3TmrFoct33rjWGfkyU/WvmlES3UzQ3qMzMbVCSvAeKL2FJLfN0ObmSkRB/x1W++kek= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror.sh env: CEPH_ARGS: '' RBD_IMAGE_FEATURES: layering RBD_MIRROR_CONFIG_KEY: '1' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-22_19:35:38 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653 2026-02-22T19:43:46.062 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa; will attempt to use it 2026-02-22T19:43:46.062 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa/tasks 2026-02-22T19:43:46.062 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-22T19:43:46.064 INFO:teuthology.task.internal:Checking packages... 2026-02-22T19:43:46.064 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'f90f4eaebb28b619cf76e80cbbe997656da69579' 2026-02-22T19:43:46.064 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-22T19:43:46.064 INFO:teuthology.packaging:ref: None 2026-02-22T19:43:46.064 INFO:teuthology.packaging:tag: None 2026-02-22T19:43:46.064 INFO:teuthology.packaging:branch: wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T19:43:46.064 INFO:teuthology.packaging:sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T19:43:46.064 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T19:43:46.202 INFO:teuthology.task.internal:Found packages for ceph version 19.2.3-689-gf90f4eae-1jammy 2026-02-22T19:43:46.203 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-22T19:43:46.215 INFO:teuthology.task.internal:no buildpackages task found 2026-02-22T19:43:46.216 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-22T19:43:46.228 INFO:teuthology.task.internal:Saving configuration 2026-02-22T19:43:46.236 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-22T19:43:46.248 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-22T19:43:46.318 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial012.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 19:38:19.450196', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOe6dLRZ/gdqJbhdZIyvU54wFPit2nEefIsaOEVDf1CI'} 2026-02-22T19:43:46.386 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial098.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 19:38:19.450832', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM9Xpu9dto9xaXN1+Q2dYxMjJHlmmFJRBE0Yb9O9I8nS'} 2026-02-22T19:43:46.386 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-22T19:43:46.397 INFO:teuthology.task.internal:roles: ubuntu@trial012.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-22T19:43:46.397 INFO:teuthology.task.internal:roles: ubuntu@trial098.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-22T19:43:46.397 INFO:teuthology.run_tasks:Running task console_log... 2026-02-22T19:43:46.546 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7819f2545800>, signals=[15]) 2026-02-22T19:43:46.546 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-22T19:43:46.556 INFO:teuthology.task.internal:Opening connections... 2026-02-22T19:43:46.556 DEBUG:teuthology.task.internal:connecting to ubuntu@trial012.front.sepia.ceph.com 2026-02-22T19:43:46.557 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T19:43:46.627 DEBUG:teuthology.task.internal:connecting to ubuntu@trial098.front.sepia.ceph.com 2026-02-22T19:43:46.628 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T19:43:46.692 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-22T19:43:46.709 DEBUG:teuthology.orchestra.run.trial012:> uname -m 2026-02-22T19:43:46.712 INFO:teuthology.orchestra.run.trial012.stdout:x86_64 2026-02-22T19:43:46.713 DEBUG:teuthology.orchestra.run.trial012:> cat /etc/os-release 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:NAME="Ubuntu" 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:VERSION_ID="22.04" 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:VERSION_CODENAME=jammy 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:ID=ubuntu 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:ID_LIKE=debian 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-22T19:43:46.760 INFO:teuthology.orchestra.run.trial012.stdout:UBUNTU_CODENAME=jammy 2026-02-22T19:43:46.761 INFO:teuthology.lock.ops:Updating trial012.front.sepia.ceph.com on lock server 2026-02-22T19:43:46.824 DEBUG:teuthology.orchestra.run.trial098:> uname -m 2026-02-22T19:43:46.827 INFO:teuthology.orchestra.run.trial098.stdout:x86_64 2026-02-22T19:43:46.827 DEBUG:teuthology.orchestra.run.trial098:> cat /etc/os-release 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:NAME="Ubuntu" 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:VERSION_ID="22.04" 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:VERSION_CODENAME=jammy 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:ID=ubuntu 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:ID_LIKE=debian 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-22T19:43:46.873 INFO:teuthology.orchestra.run.trial098.stdout:UBUNTU_CODENAME=jammy 2026-02-22T19:43:46.873 INFO:teuthology.lock.ops:Updating trial098.front.sepia.ceph.com on lock server 2026-02-22T19:43:46.940 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-22T19:43:46.954 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-22T19:43:46.968 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-22T19:43:46.969 DEBUG:teuthology.orchestra.run.trial012:> test '!' -e /home/ubuntu/cephtest 2026-02-22T19:43:46.970 DEBUG:teuthology.orchestra.run.trial098:> test '!' -e /home/ubuntu/cephtest 2026-02-22T19:43:46.973 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-22T19:43:46.984 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-22T19:43:46.984 DEBUG:teuthology.orchestra.run.trial012:> test -z $(ls -A /var/lib/ceph) 2026-02-22T19:43:47.021 DEBUG:teuthology.orchestra.run.trial098:> test -z $(ls -A /var/lib/ceph) 2026-02-22T19:43:47.024 INFO:teuthology.orchestra.run.trial012.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-22T19:43:47.025 INFO:teuthology.orchestra.run.trial098.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-22T19:43:47.025 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-22T19:43:47.159 INFO:teuthology.run_tasks:Running task kernel... 2026-02-22T19:43:47.180 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-22T19:43:47.181 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-22T19:43:47.181 DEBUG:teuthology.orchestra.run.trial012:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T19:43:47.181 DEBUG:teuthology.orchestra.run.trial098:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T19:43:47.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T19:43:47.184 DEBUG:teuthology.orchestra.run.trial098:> uname -r 2026-02-22T19:43:47.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T19:43:47.184 DEBUG:teuthology.orchestra.run.trial012:> uname -r 2026-02-22T19:43:47.229 INFO:teuthology.orchestra.run.trial098.stdout:5.15.0-170-generic 2026-02-22T19:43:47.229 INFO:teuthology.task.kernel:Running kernel on trial098: 5.15.0-170-generic 2026-02-22T19:43:47.229 DEBUG:teuthology.orchestra.run.trial098:> sudo apt-get clean 2026-02-22T19:43:47.232 INFO:teuthology.orchestra.run.trial012.stdout:5.15.0-170-generic 2026-02-22T19:43:47.232 INFO:teuthology.task.kernel:Running kernel on trial012: 5.15.0-170-generic 2026-02-22T19:43:47.232 DEBUG:teuthology.orchestra.run.trial012:> sudo apt-get clean 2026-02-22T19:43:47.286 DEBUG:teuthology.orchestra.run.trial098:> sudo apt-get update 2026-02-22T19:43:47.290 DEBUG:teuthology.orchestra.run.trial012:> sudo apt-get update 2026-02-22T19:43:47.791 INFO:teuthology.orchestra.run.trial098.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-22T19:43:47.794 INFO:teuthology.orchestra.run.trial012.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-22T19:43:47.880 INFO:teuthology.orchestra.run.trial098.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-22T19:43:47.884 INFO:teuthology.orchestra.run.trial012.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-22T19:43:48.216 INFO:teuthology.orchestra.run.trial098.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-22T19:43:48.221 INFO:teuthology.orchestra.run.trial012.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-22T19:43:48.363 INFO:teuthology.orchestra.run.trial098.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-22T19:43:48.369 INFO:teuthology.orchestra.run.trial012.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-22T19:43:48.503 INFO:teuthology.orchestra.run.trial098.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3245 kB] 2026-02-22T19:43:48.508 INFO:teuthology.orchestra.run.trial012.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3245 kB] 2026-02-22T19:43:48.597 INFO:teuthology.orchestra.run.trial012.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [495 kB] 2026-02-22T19:43:48.609 INFO:teuthology.orchestra.run.trial012.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.1 kB] 2026-02-22T19:43:48.619 INFO:teuthology.orchestra.run.trial012.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-22T19:43:48.636 INFO:teuthology.orchestra.run.trial012.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-22T19:43:48.648 INFO:teuthology.orchestra.run.trial012.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [30.3 kB] 2026-02-22T19:43:48.658 INFO:teuthology.orchestra.run.trial012.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5310 kB] 2026-02-22T19:43:48.698 INFO:teuthology.orchestra.run.trial012.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1005 kB] 2026-02-22T19:43:48.714 INFO:teuthology.orchestra.run.trial012.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2983 kB] 2026-02-22T19:43:48.740 INFO:teuthology.orchestra.run.trial012.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/main Translation-en [427 kB] 2026-02-22T19:43:48.753 INFO:teuthology.orchestra.run.trial012.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-22T19:43:48.763 INFO:teuthology.orchestra.run.trial012.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1018 kB] 2026-02-22T19:43:48.778 INFO:teuthology.orchestra.run.trial012.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [22.7 kB] 2026-02-22T19:43:48.788 INFO:teuthology.orchestra.run.trial012.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [5147 kB] 2026-02-22T19:43:48.827 INFO:teuthology.orchestra.run.trial012.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy-security/restricted Translation-en [979 kB] 2026-02-22T19:43:48.903 INFO:teuthology.orchestra.run.trial098.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [495 kB] 2026-02-22T19:43:48.916 INFO:teuthology.orchestra.run.trial098.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.1 kB] 2026-02-22T19:43:48.916 INFO:teuthology.orchestra.run.trial098.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-22T19:43:48.951 INFO:teuthology.orchestra.run.trial098.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-22T19:43:48.958 INFO:teuthology.orchestra.run.trial098.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [30.3 kB] 2026-02-22T19:43:48.959 INFO:teuthology.orchestra.run.trial098.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5310 kB] 2026-02-22T19:43:49.115 INFO:teuthology.orchestra.run.trial098.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1005 kB] 2026-02-22T19:43:49.138 INFO:teuthology.orchestra.run.trial098.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2983 kB] 2026-02-22T19:43:49.226 INFO:teuthology.orchestra.run.trial098.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/main Translation-en [427 kB] 2026-02-22T19:43:49.234 INFO:teuthology.orchestra.run.trial098.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-22T19:43:49.235 INFO:teuthology.orchestra.run.trial098.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1018 kB] 2026-02-22T19:43:49.268 INFO:teuthology.orchestra.run.trial098.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [22.7 kB] 2026-02-22T19:43:49.270 INFO:teuthology.orchestra.run.trial098.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [5147 kB] 2026-02-22T19:43:49.417 INFO:teuthology.orchestra.run.trial098.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy-security/restricted Translation-en [979 kB] 2026-02-22T19:43:52.516 INFO:teuthology.orchestra.run.trial012.stdout:Fetched 22.6 MB in 3s (8806 kB/s) 2026-02-22T19:43:52.857 INFO:teuthology.orchestra.run.trial098.stdout:Fetched 22.6 MB in 3s (7774 kB/s) 2026-02-22T19:43:53.039 INFO:teuthology.orchestra.run.trial012.stdout:Reading package lists... 2026-02-22T19:43:53.052 DEBUG:teuthology.orchestra.run.trial012:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-22T19:43:53.084 INFO:teuthology.orchestra.run.trial012.stdout:Reading package lists... 2026-02-22T19:43:53.190 INFO:teuthology.orchestra.run.trial012.stdout:Building dependency tree... 2026-02-22T19:43:53.191 INFO:teuthology.orchestra.run.trial012.stdout:Reading state information... 2026-02-22T19:43:53.276 INFO:teuthology.orchestra.run.trial012.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-22T19:43:53.276 INFO:teuthology.orchestra.run.trial012.stdout:linux-image-generic set to manually installed. 2026-02-22T19:43:53.276 INFO:teuthology.orchestra.run.trial012.stdout:The following packages were automatically installed and are no longer required: 2026-02-22T19:43:53.276 INFO:teuthology.orchestra.run.trial012.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-22T19:43:53.276 INFO:teuthology.orchestra.run.trial012.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-22T19:43:53.276 INFO:teuthology.orchestra.run.trial012.stdout: sg3-utils-udev 2026-02-22T19:43:53.276 INFO:teuthology.orchestra.run.trial012.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-22T19:43:53.295 INFO:teuthology.orchestra.run.trial012.stdout:0 upgraded, 0 newly installed, 0 to remove and 13 not upgraded. 2026-02-22T19:43:53.297 DEBUG:teuthology.orchestra.run.trial012:> dpkg -s linux-image-generic 2026-02-22T19:43:53.353 INFO:teuthology.orchestra.run.trial012.stdout:Package: linux-image-generic 2026-02-22T19:43:53.353 INFO:teuthology.orchestra.run.trial012.stdout:Status: install ok installed 2026-02-22T19:43:53.353 INFO:teuthology.orchestra.run.trial012.stdout:Priority: optional 2026-02-22T19:43:53.353 INFO:teuthology.orchestra.run.trial012.stdout:Section: kernel 2026-02-22T19:43:53.353 INFO:teuthology.orchestra.run.trial012.stdout:Installed-Size: 21 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout:Maintainer: Ubuntu Kernel Team 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout:Architecture: amd64 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout:Source: linux-meta 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout:Version: 5.15.0.170.159 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout:Recommends: thermald 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout:Description: Generic Linux kernel image 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout: This package will always depend on the latest generic kernel image 2026-02-22T19:43:53.354 INFO:teuthology.orchestra.run.trial012.stdout: available. 2026-02-22T19:43:53.354 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T19:43:53.354 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-22T19:43:53.354 DEBUG:teuthology.orchestra.run.trial012:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T19:43:53.392 INFO:teuthology.orchestra.run.trial098.stdout:Reading package lists... 2026-02-22T19:43:53.405 DEBUG:teuthology.orchestra.run.trial098:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-22T19:43:53.408 INFO:teuthology.orchestra.run.trial012.stdout:ttyS1 2026-02-22T19:43:53.430 DEBUG:teuthology.parallel:result is None 2026-02-22T19:43:53.438 INFO:teuthology.orchestra.run.trial098.stdout:Reading package lists... 2026-02-22T19:43:53.551 INFO:teuthology.orchestra.run.trial098.stdout:Building dependency tree... 2026-02-22T19:43:53.551 INFO:teuthology.orchestra.run.trial098.stdout:Reading state information... 2026-02-22T19:43:53.640 INFO:teuthology.orchestra.run.trial098.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-22T19:43:53.640 INFO:teuthology.orchestra.run.trial098.stdout:linux-image-generic set to manually installed. 2026-02-22T19:43:53.640 INFO:teuthology.orchestra.run.trial098.stdout:The following packages were automatically installed and are no longer required: 2026-02-22T19:43:53.640 INFO:teuthology.orchestra.run.trial098.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-22T19:43:53.640 INFO:teuthology.orchestra.run.trial098.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-22T19:43:53.640 INFO:teuthology.orchestra.run.trial098.stdout: sg3-utils-udev 2026-02-22T19:43:53.640 INFO:teuthology.orchestra.run.trial098.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-22T19:43:53.660 INFO:teuthology.orchestra.run.trial098.stdout:0 upgraded, 0 newly installed, 0 to remove and 13 not upgraded. 2026-02-22T19:43:53.662 DEBUG:teuthology.orchestra.run.trial098:> dpkg -s linux-image-generic 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Package: linux-image-generic 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Status: install ok installed 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Priority: optional 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Section: kernel 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Installed-Size: 21 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Maintainer: Ubuntu Kernel Team 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Architecture: amd64 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Source: linux-meta 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Version: 5.15.0.170.159 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Recommends: thermald 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout:Description: Generic Linux kernel image 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout: This package will always depend on the latest generic kernel image 2026-02-22T19:43:53.715 INFO:teuthology.orchestra.run.trial098.stdout: available. 2026-02-22T19:43:53.715 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T19:43:53.716 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-22T19:43:53.716 DEBUG:teuthology.orchestra.run.trial098:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T19:43:53.766 INFO:teuthology.orchestra.run.trial098.stdout:ttyS1 2026-02-22T19:43:53.783 DEBUG:teuthology.parallel:result is None 2026-02-22T19:43:53.783 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-22T19:43:53.798 INFO:teuthology.task.internal:Creating test directory... 2026-02-22T19:43:53.798 DEBUG:teuthology.orchestra.run.trial012:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T19:43:53.800 DEBUG:teuthology.orchestra.run.trial098:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T19:43:53.832 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-22T19:43:53.844 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-22T19:43:53.858 INFO:teuthology.task.internal:Creating archive directory... 2026-02-22T19:43:53.858 DEBUG:teuthology.orchestra.run.trial012:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T19:43:53.860 DEBUG:teuthology.orchestra.run.trial098:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T19:43:53.879 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-22T19:43:53.894 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-22T19:43:53.895 DEBUG:teuthology.orchestra.run.trial012:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T19:43:53.905 DEBUG:teuthology.orchestra.run.trial098:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T19:43:53.912 INFO:teuthology.orchestra.run.trial012.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T19:43:53.919 INFO:teuthology.orchestra.run.trial012.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T19:43:53.930 INFO:teuthology.orchestra.run.trial098.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T19:43:53.936 INFO:teuthology.orchestra.run.trial098.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T19:43:53.937 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-22T19:43:53.949 INFO:teuthology.task.internal:Configuring sudo... 2026-02-22T19:43:53.950 DEBUG:teuthology.orchestra.run.trial012:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T19:43:53.965 DEBUG:teuthology.orchestra.run.trial098:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T19:43:53.989 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-22T19:43:54.001 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-22T19:43:54.002 DEBUG:teuthology.orchestra.run.trial012:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T19:43:54.013 DEBUG:teuthology.orchestra.run.trial098:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T19:43:54.032 DEBUG:teuthology.orchestra.run.trial012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T19:43:54.063 DEBUG:teuthology.orchestra.run.trial012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T19:43:54.107 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:43:54.107 DEBUG:teuthology.orchestra.run.trial012:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T19:43:54.161 DEBUG:teuthology.orchestra.run.trial098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T19:43:54.164 DEBUG:teuthology.orchestra.run.trial098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T19:43:54.209 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:43:54.209 DEBUG:teuthology.orchestra.run.trial098:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T19:43:54.258 DEBUG:teuthology.orchestra.run.trial012:> sudo service rsyslog restart 2026-02-22T19:43:54.259 DEBUG:teuthology.orchestra.run.trial098:> sudo service rsyslog restart 2026-02-22T19:43:54.319 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-22T19:43:54.332 INFO:teuthology.task.internal:Starting timer... 2026-02-22T19:43:54.332 INFO:teuthology.run_tasks:Running task pcp... 2026-02-22T19:43:54.347 INFO:teuthology.run_tasks:Running task selinux... 2026-02-22T19:43:54.360 DEBUG:teuthology.task.selinux:Excluding trial012: OS 'ubuntu' does not support SELinux 2026-02-22T19:43:54.360 DEBUG:teuthology.task.selinux:Excluding trial098: OS 'ubuntu' does not support SELinux 2026-02-22T19:43:54.360 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-22T19:43:54.360 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2026-02-22T19:43:54.360 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-22T19:43:54.360 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-22T19:43:54.382 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-22T19:43:54.389 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-22T19:43:54.390 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial012.front.sepia.ceph.com,trial098.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-22T19:46:02.067 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial012.front.sepia.ceph.com'), Remote(name='ubuntu@trial098.front.sepia.ceph.com')] 2026-02-22T19:46:02.069 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial012.front.sepia.ceph.com' 2026-02-22T19:46:02.071 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T19:46:02.138 DEBUG:teuthology.orchestra.run.trial012:> true 2026-02-22T19:46:02.386 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial012.front.sepia.ceph.com' 2026-02-22T19:46:02.387 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial098.front.sepia.ceph.com' 2026-02-22T19:46:02.387 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T19:46:02.452 DEBUG:teuthology.orchestra.run.trial098:> true 2026-02-22T19:46:02.701 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial098.front.sepia.ceph.com' 2026-02-22T19:46:02.702 INFO:teuthology.run_tasks:Running task clock... 2026-02-22T19:46:02.715 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-22T19:46:02.715 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T19:46:02.715 DEBUG:teuthology.orchestra.run.trial012:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T19:46:02.717 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T19:46:02.717 DEBUG:teuthology.orchestra.run.trial098:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T19:46:02.734 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-22T19:46:02.734 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: Command line: ntpd -gq 2026-02-22T19:46:02.734 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: ---------------------------------------------------- 2026-02-22T19:46:02.734 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: ntp-4 is maintained by Network Time Foundation, 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: corporation. Support and training for ntp-4 are 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: available at https://www.nwtime.org/support 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: ---------------------------------------------------- 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: proto: precision = 0.020 usec (-25) 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: basedate set to 2022-02-04 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: gps base set to 2022-02-06 (week 2196) 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-22T19:46:02.735 INFO:teuthology.orchestra.run.trial012.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-22T19:46:02.736 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: restrict ::: KOD does nothing without LIMITED. 2026-02-22T19:46:02.736 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: Listen and drop on 0 v6wildcard [::]:123 2026-02-22T19:46:02.736 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-22T19:46:02.736 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: Listen normally on 2 lo 127.0.0.1:123 2026-02-22T19:46:02.736 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: Listen normally on 3 enp1s0f0 10.20.193.12:123 2026-02-22T19:46:02.736 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: Listen normally on 4 lo [::1]:123 2026-02-22T19:46:02.736 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:6242%2]:123 2026-02-22T19:46:02.736 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:02 ntpd[9331]: Listening on routing socket on fd #22 for interface updates 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: Command line: ntpd -gq 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: ---------------------------------------------------- 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: ntp-4 is maintained by Network Time Foundation, 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: corporation. Support and training for ntp-4 are 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: available at https://www.nwtime.org/support 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: ---------------------------------------------------- 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: proto: precision = 0.030 usec (-25) 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: basedate set to 2022-02-04 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: gps base set to 2022-02-06 (week 2196) 2026-02-22T19:46:02.767 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: restrict ::: KOD does nothing without LIMITED. 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: Listen and drop on 0 v6wildcard [::]:123 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: Listen normally on 2 lo 127.0.0.1:123 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: Listen normally on 3 enp1s0f0 10.20.193.98:123 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: Listen normally on 4 lo [::1]:123 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:61ec%2]:123 2026-02-22T19:46:02.768 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:02 ntpd[9416]: Listening on routing socket on fd #22 for interface updates 2026-02-22T19:46:03.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:03 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:03.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:03 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:03.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:03 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:03.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:03 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:03.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:03 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:03.738 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:03 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:03.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:03 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:03.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:03 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:03.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:03 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:03.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:03 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:03.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:03 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:03.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:03 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:04.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:04.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:04.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:04.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:04.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:04.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:04.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:04.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:04.764 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:04.764 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:04.764 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:04.764 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:04 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:04.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:04 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:04.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:04 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:04.821 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:04 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:04.821 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:04 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:04.821 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:04 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:04.821 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:04 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:05.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:05.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:05.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:05.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:05.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:05.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:05.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:05.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:05.752 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:05.752 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:05.752 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:05.752 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:05 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:05.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:05 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:05.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:05 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:05.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:05 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:05.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:05 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:05.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:05 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:05.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:05 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:06.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:06.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:06.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:06.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:06.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:06.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:06.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:06.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:06.763 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:06.763 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:06.764 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:06.764 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:06 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:06.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:06.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:06.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:06.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:06.770 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:06.770 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:06.771 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:06.771 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:06.781 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:06.781 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:06.781 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:06.781 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:06 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:07.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:07.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:07.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:07.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:07.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:07.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:07.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:07.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:07.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:07.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:07.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:07.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:07.779 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:07.779 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:07.779 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:07.779 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:07 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:07.795 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:07.795 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:07.795 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:07.795 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:07.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:07.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:07.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:07.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:07 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:08.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:08.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:08.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:08.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:08.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:08.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:08.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:08.761 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:08.763 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:08.764 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:08.764 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:08.764 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:08 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:08.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:08.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:08.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:08.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:08.770 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:08.770 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:08.770 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:08.770 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:08.823 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:08.823 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:08.823 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:08.824 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:08 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:09.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:09.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:09.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:09.734 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:09.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:09.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:09.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:09.737 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:09.737 INFO:teuthology.orchestra.run.trial012.stdout:22 Feb 19:46:09 ntpd[9331]: ntpd: time set +0.524318 s 2026-02-22T19:46:09.737 INFO:teuthology.orchestra.run.trial012.stdout:ntpd: time set +0.524318s 2026-02-22T19:46:09.738 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-22T19:46:09.738 INFO:teuthology.orchestra.run.trial012.stderr:22 Feb 19:46:09 ntpd[9331]: can't open /var/log/ntpstats/loopstats.20260222: Permission denied 2026-02-22T19:46:09.760 INFO:teuthology.orchestra.run.trial012.stdout:No association ID's returned 2026-02-22T19:46:09.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:09.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:09.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:09.767 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:09.794 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:09.794 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:09.795 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:09.795 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:09.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T19:46:09.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T19:46:09.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T19:46:09.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T19:46:09.797 INFO:teuthology.orchestra.run.trial098.stdout:22 Feb 19:46:09 ntpd[9416]: ntpd: time set +0.245349 s 2026-02-22T19:46:09.797 INFO:teuthology.orchestra.run.trial098.stdout:ntpd: time set +0.245349s 2026-02-22T19:46:09.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-22T19:46:09.797 INFO:teuthology.orchestra.run.trial098.stderr:22 Feb 19:46:09 ntpd[9416]: can't open /var/log/ntpstats/loopstats.20260222: Permission denied 2026-02-22T19:46:09.856 INFO:teuthology.orchestra.run.trial098.stdout: remote refid st t when poll reach delay offset jitter 2026-02-22T19:46:09.857 INFO:teuthology.orchestra.run.trial098.stdout:============================================================================== 2026-02-22T19:46:09.857 INFO:teuthology.orchestra.run.trial098.stdout: 23.186.168.130 .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T19:46:09.857 INFO:teuthology.orchestra.run.trial098.stdout: 172-234-25-10.i .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T19:46:09.857 INFO:teuthology.orchestra.run.trial098.stdout: 69.172.133.130 .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T19:46:09.857 INFO:teuthology.orchestra.run.trial098.stdout: owners.kjsl.com .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T19:46:09.857 INFO:teuthology.run_tasks:Running task install... 2026-02-22T19:46:09.868 DEBUG:teuthology.task.install:project ceph 2026-02-22T19:46:09.868 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}} 2026-02-22T19:46:09.868 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'} 2026-02-22T19:46:09.868 INFO:teuthology.task.install:Using flavor: default 2026-02-22T19:46:09.871 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-22T19:46:09.871 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-22T19:46:09.871 DEBUG:teuthology.orchestra.run.trial012:> sudo apt-key list | grep Ceph 2026-02-22T19:46:09.872 DEBUG:teuthology.orchestra.run.trial098:> sudo apt-key list | grep Ceph 2026-02-22T19:46:09.923 INFO:teuthology.orchestra.run.trial012.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-22T19:46:09.951 INFO:teuthology.orchestra.run.trial098.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-22T19:46:09.951 INFO:teuthology.orchestra.run.trial012.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-22T19:46:09.951 INFO:teuthology.orchestra.run.trial012.stdout:uid [ unknown] Ceph.com (release key) 2026-02-22T19:46:09.952 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-22T19:46:09.952 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T19:46:10.613 INFO:teuthology.orchestra.run.trial098.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-22T19:46:10.613 INFO:teuthology.orchestra.run.trial098.stdout:uid [ unknown] Ceph.com (release key) 2026-02-22T19:46:10.613 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-22T19:46:10.613 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T19:46:10.765 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default/ 2026-02-22T19:46:10.766 INFO:teuthology.task.install.deb:Package version is 19.2.3-689-gf90f4eae-1jammy 2026-02-22T19:46:10.778 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default/ 2026-02-22T19:46:10.779 INFO:teuthology.task.install.deb:Package version is 19.2.3-689-gf90f4eae-1jammy 2026-02-22T19:46:11.079 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:46:11.079 DEBUG:teuthology.orchestra.run.trial012:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-22T19:46:11.089 DEBUG:teuthology.orchestra.run.trial012:> sudo apt-get update 2026-02-22T19:46:11.131 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:46:11.131 DEBUG:teuthology.orchestra.run.trial098:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-22T19:46:11.140 DEBUG:teuthology.orchestra.run.trial098:> sudo apt-get update 2026-02-22T19:46:11.272 INFO:teuthology.orchestra.run.trial012.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-22T19:46:11.277 INFO:teuthology.orchestra.run.trial012.stdout:Hit:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-22T19:46:11.287 INFO:teuthology.orchestra.run.trial012.stdout:Hit:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-22T19:46:11.297 INFO:teuthology.orchestra.run.trial012.stdout:Hit:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-22T19:46:11.435 INFO:teuthology.orchestra.run.trial012.stdout:Ign:5 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy InRelease 2026-02-22T19:46:11.479 INFO:teuthology.orchestra.run.trial098.stdout:Ign:1 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy InRelease 2026-02-22T19:46:11.492 INFO:teuthology.orchestra.run.trial012.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy Release [7662 B] 2026-02-22T19:46:11.535 INFO:teuthology.orchestra.run.trial098.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy Release [7662 B] 2026-02-22T19:46:11.546 INFO:teuthology.orchestra.run.trial012.stdout:Ign:7 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-22T19:46:11.574 INFO:teuthology.orchestra.run.trial098.stdout:Hit:3 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-22T19:46:11.599 INFO:teuthology.orchestra.run.trial098.stdout:Ign:4 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-22T19:46:11.609 INFO:teuthology.orchestra.run.trial012.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.1 kB] 2026-02-22T19:46:11.643 INFO:teuthology.orchestra.run.trial098.stdout:Hit:5 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-22T19:46:11.667 INFO:teuthology.orchestra.run.trial098.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.1 kB] 2026-02-22T19:46:11.712 INFO:teuthology.orchestra.run.trial098.stdout:Hit:7 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-22T19:46:11.767 INFO:teuthology.orchestra.run.trial012.stdout:Fetched 25.8 kB in 1s (50.1 kB/s) 2026-02-22T19:46:11.785 INFO:teuthology.orchestra.run.trial098.stdout:Hit:8 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-22T19:46:11.931 INFO:teuthology.orchestra.run.trial098.stdout:Fetched 25.8 kB in 1s (41.0 kB/s) 2026-02-22T19:46:12.280 INFO:teuthology.orchestra.run.trial012.stdout:Reading package lists... 2026-02-22T19:46:12.294 DEBUG:teuthology.orchestra.run.trial012:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.2.3-689-gf90f4eae-1jammy cephadm=19.2.3-689-gf90f4eae-1jammy ceph-mds=19.2.3-689-gf90f4eae-1jammy ceph-mgr=19.2.3-689-gf90f4eae-1jammy ceph-common=19.2.3-689-gf90f4eae-1jammy ceph-fuse=19.2.3-689-gf90f4eae-1jammy ceph-test=19.2.3-689-gf90f4eae-1jammy ceph-volume=19.2.3-689-gf90f4eae-1jammy radosgw=19.2.3-689-gf90f4eae-1jammy python3-rados=19.2.3-689-gf90f4eae-1jammy python3-rgw=19.2.3-689-gf90f4eae-1jammy python3-cephfs=19.2.3-689-gf90f4eae-1jammy python3-rbd=19.2.3-689-gf90f4eae-1jammy libcephfs2=19.2.3-689-gf90f4eae-1jammy libcephfs-dev=19.2.3-689-gf90f4eae-1jammy librados2=19.2.3-689-gf90f4eae-1jammy librbd1=19.2.3-689-gf90f4eae-1jammy rbd-fuse=19.2.3-689-gf90f4eae-1jammy rbd-mirror=19.2.3-689-gf90f4eae-1jammy 2026-02-22T19:46:12.327 INFO:teuthology.orchestra.run.trial012.stdout:Reading package lists... 2026-02-22T19:46:12.440 INFO:teuthology.orchestra.run.trial012.stdout:Building dependency tree... 2026-02-22T19:46:12.440 INFO:teuthology.orchestra.run.trial012.stdout:Reading state information... 2026-02-22T19:46:12.448 INFO:teuthology.orchestra.run.trial098.stdout:Reading package lists... 2026-02-22T19:46:12.461 DEBUG:teuthology.orchestra.run.trial098:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.2.3-689-gf90f4eae-1jammy cephadm=19.2.3-689-gf90f4eae-1jammy ceph-mds=19.2.3-689-gf90f4eae-1jammy ceph-mgr=19.2.3-689-gf90f4eae-1jammy ceph-common=19.2.3-689-gf90f4eae-1jammy ceph-fuse=19.2.3-689-gf90f4eae-1jammy ceph-test=19.2.3-689-gf90f4eae-1jammy ceph-volume=19.2.3-689-gf90f4eae-1jammy radosgw=19.2.3-689-gf90f4eae-1jammy python3-rados=19.2.3-689-gf90f4eae-1jammy python3-rgw=19.2.3-689-gf90f4eae-1jammy python3-cephfs=19.2.3-689-gf90f4eae-1jammy python3-rbd=19.2.3-689-gf90f4eae-1jammy libcephfs2=19.2.3-689-gf90f4eae-1jammy libcephfs-dev=19.2.3-689-gf90f4eae-1jammy librados2=19.2.3-689-gf90f4eae-1jammy librbd1=19.2.3-689-gf90f4eae-1jammy rbd-fuse=19.2.3-689-gf90f4eae-1jammy rbd-mirror=19.2.3-689-gf90f4eae-1jammy 2026-02-22T19:46:12.494 INFO:teuthology.orchestra.run.trial098.stdout:Reading package lists... 2026-02-22T19:46:12.534 INFO:teuthology.orchestra.run.trial012.stdout:The following packages were automatically installed and are no longer required: 2026-02-22T19:46:12.534 INFO:teuthology.orchestra.run.trial012.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-22T19:46:12.534 INFO:teuthology.orchestra.run.trial012.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-22T19:46:12.534 INFO:teuthology.orchestra.run.trial012.stdout: sg3-utils-udev 2026-02-22T19:46:12.534 INFO:teuthology.orchestra.run.trial012.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout:The following additional packages will be installed: 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 liblua5.3-dev libnbd0 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: libradosstriper1 librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: libthrift-0.16.0 lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: pkg-config python-asyncssh-doc python-pastedeploy-tpl python3-asyncssh 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cheroot 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-logutils 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-mako python3-natsort python3-paste python3-pastedeploy 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-pyinotify python3-pytest python3-repoze.lru 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplegeneric 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-simplejson python3-singledispatch python3-sklearn 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-sklearn-lib python3-tempita python3-tempora python3-threadpoolctl 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-toml python3-waitress python3-wcwidth python3-webob 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-websocket python3-webtest python3-werkzeug python3-zc.lockfile 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: qttranslations5-l10n socat unzip xmlstarlet zip 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout:Suggested packages: 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python3-influxdb readline-doc python3-beaker python-mako-doc 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python-natsort-doc httpd-wsgi libapache2-mod-python libapache2-mod-scgi 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: libjs-mochikit python-pecan-doc python-psutil-doc subversion 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python-pygments-doc ttf-bitstream-vera python-pyinotify-doc python3-dap 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout: python-webtest-doc python-werkzeug-doc python3-watchdog 2026-02-22T19:46:12.535 INFO:teuthology.orchestra.run.trial012.stdout:Recommended packages: 2026-02-22T19:46:12.536 INFO:teuthology.orchestra.run.trial012.stdout: btrfs-tools 2026-02-22T19:46:12.571 INFO:teuthology.orchestra.run.trial012.stdout:The following NEW packages will be installed: 2026-02-22T19:46:12.571 INFO:teuthology.orchestra.run.trial012.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-22T19:46:12.571 INFO:teuthology.orchestra.run.trial012.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-22T19:46:12.571 INFO:teuthology.orchestra.run.trial012.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-02-22T19:46:12.571 INFO:teuthology.orchestra.run.trial012.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 2026-02-22T19:46:12.571 INFO:teuthology.orchestra.run.trial012.stdout: liblua5.3-dev libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-02-22T19:46:12.571 INFO:teuthology.orchestra.run.trial012.stdout: libqt5dbus5 libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1 2026-02-22T19:46:12.571 INFO:teuthology.orchestra.run.trial012.stdout: libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python-pastedeploy-tpl python3-asyncssh python3-cachetools 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-ceph-argparse python3-ceph-common python3-cephfs python3-cheroot 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-logutils 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-mako python3-natsort python3-paste python3-pastedeploy 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-pyinotify python3-pytest python3-rados python3-rbd 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-repoze.lru python3-requests-oauthlib python3-rgw python3-routes 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-threadpoolctl python3-toml python3-waitress python3-wcwidth 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-webob python3-websocket python3-webtest python3-werkzeug 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse rbd-mirror socat 2026-02-22T19:46:12.572 INFO:teuthology.orchestra.run.trial012.stdout: unzip xmlstarlet zip 2026-02-22T19:46:12.606 INFO:teuthology.orchestra.run.trial098.stdout:Building dependency tree... 2026-02-22T19:46:12.606 INFO:teuthology.orchestra.run.trial098.stdout:Reading state information... 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout:The following packages were automatically installed and are no longer required: 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout: sg3-utils-udev 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout:The following additional packages will be installed: 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 liblua5.3-dev libnbd0 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-02-22T19:46:12.696 INFO:teuthology.orchestra.run.trial098.stdout: libradosstriper1 librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: libthrift-0.16.0 lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: pkg-config python-asyncssh-doc python-pastedeploy-tpl python3-asyncssh 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cheroot 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-logutils 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-mako python3-natsort python3-paste python3-pastedeploy 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-pyinotify python3-pytest python3-repoze.lru 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplegeneric 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-simplejson python3-singledispatch python3-sklearn 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-sklearn-lib python3-tempita python3-tempora python3-threadpoolctl 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-toml python3-waitress python3-wcwidth python3-webob 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-websocket python3-webtest python3-werkzeug python3-zc.lockfile 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: qttranslations5-l10n socat unzip xmlstarlet zip 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout:Suggested packages: 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python3-influxdb readline-doc python3-beaker python-mako-doc 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python-natsort-doc httpd-wsgi libapache2-mod-python libapache2-mod-scgi 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: libjs-mochikit python-pecan-doc python-psutil-doc subversion 2026-02-22T19:46:12.697 INFO:teuthology.orchestra.run.trial098.stdout: python-pygments-doc ttf-bitstream-vera python-pyinotify-doc python3-dap 2026-02-22T19:46:12.698 INFO:teuthology.orchestra.run.trial098.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2026-02-22T19:46:12.698 INFO:teuthology.orchestra.run.trial098.stdout: python-webtest-doc python-werkzeug-doc python3-watchdog 2026-02-22T19:46:12.698 INFO:teuthology.orchestra.run.trial098.stdout:Recommended packages: 2026-02-22T19:46:12.698 INFO:teuthology.orchestra.run.trial098.stdout: btrfs-tools 2026-02-22T19:46:12.732 INFO:teuthology.orchestra.run.trial098.stdout:The following NEW packages will be installed: 2026-02-22T19:46:12.732 INFO:teuthology.orchestra.run.trial098.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-22T19:46:12.732 INFO:teuthology.orchestra.run.trial098.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-22T19:46:12.732 INFO:teuthology.orchestra.run.trial098.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-02-22T19:46:12.732 INFO:teuthology.orchestra.run.trial098.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 2026-02-22T19:46:12.732 INFO:teuthology.orchestra.run.trial098.stdout: liblua5.3-dev libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-02-22T19:46:12.732 INFO:teuthology.orchestra.run.trial098.stdout: libqt5dbus5 libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python-pastedeploy-tpl python3-asyncssh python3-cachetools 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-ceph-argparse python3-ceph-common python3-cephfs python3-cheroot 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-logutils 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-mako python3-natsort python3-paste python3-pastedeploy 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-pyinotify python3-pytest python3-rados python3-rbd 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-repoze.lru python3-requests-oauthlib python3-rgw python3-routes 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-threadpoolctl python3-toml python3-waitress python3-wcwidth 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-webob python3-websocket python3-webtest python3-werkzeug 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse rbd-mirror socat 2026-02-22T19:46:12.733 INFO:teuthology.orchestra.run.trial098.stdout: unzip xmlstarlet zip 2026-02-22T19:46:12.860 INFO:teuthology.orchestra.run.trial012.stdout:0 upgraded, 109 newly installed, 0 to remove and 13 not upgraded. 2026-02-22T19:46:12.860 INFO:teuthology.orchestra.run.trial012.stdout:Need to get 181 MB of archives. 2026-02-22T19:46:12.860 INFO:teuthology.orchestra.run.trial012.stdout:After this operation, 821 MB of additional disk space will be used. 2026-02-22T19:46:12.860 INFO:teuthology.orchestra.run.trial012.stdout:Get:1 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.2.3-689-gf90f4eae-1jammy [3597 kB] 2026-02-22T19:46:12.927 INFO:teuthology.orchestra.run.trial012.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-22T19:46:13.023 INFO:teuthology.orchestra.run.trial098.stdout:0 upgraded, 109 newly installed, 0 to remove and 13 not upgraded. 2026-02-22T19:46:13.023 INFO:teuthology.orchestra.run.trial098.stdout:Need to get 181 MB of archives. 2026-02-22T19:46:13.023 INFO:teuthology.orchestra.run.trial098.stdout:After this operation, 821 MB of additional disk space will be used. 2026-02-22T19:46:13.023 INFO:teuthology.orchestra.run.trial098.stdout:Get:1 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.2.3-689-gf90f4eae-1jammy [3597 kB] 2026-02-22T19:46:13.150 INFO:teuthology.orchestra.run.trial098.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-22T19:46:13.264 INFO:teuthology.orchestra.run.trial012.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-22T19:46:13.275 INFO:teuthology.orchestra.run.trial012.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-22T19:46:13.345 INFO:teuthology.orchestra.run.trial012.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-22T19:46:13.551 INFO:teuthology.orchestra.run.trial012.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-22T19:46:13.562 INFO:teuthology.orchestra.run.trial098.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-22T19:46:13.563 INFO:teuthology.orchestra.run.trial012.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-22T19:46:13.575 INFO:teuthology.orchestra.run.trial098.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-22T19:46:13.590 INFO:teuthology.orchestra.run.trial012.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-22T19:46:13.601 INFO:teuthology.orchestra.run.trial012.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-22T19:46:13.601 INFO:teuthology.orchestra.run.trial012.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-22T19:46:13.602 INFO:teuthology.orchestra.run.trial012.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-22T19:46:13.602 INFO:teuthology.orchestra.run.trial012.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-22T19:46:13.621 INFO:teuthology.orchestra.run.trial012.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB] 2026-02-22T19:46:13.622 INFO:teuthology.orchestra.run.trial012.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB] 2026-02-22T19:46:13.626 INFO:teuthology.orchestra.run.trial012.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB] 2026-02-22T19:46:13.660 INFO:teuthology.orchestra.run.trial098.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-22T19:46:13.696 INFO:teuthology.orchestra.run.trial012.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5034 B] 2026-02-22T19:46:13.696 INFO:teuthology.orchestra.run.trial012.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB] 2026-02-22T19:46:13.699 INFO:teuthology.orchestra.run.trial012.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB] 2026-02-22T19:46:13.701 INFO:teuthology.orchestra.run.trial012.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB] 2026-02-22T19:46:13.703 INFO:teuthology.orchestra.run.trial012.stdout:Get:20 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-22T19:46:13.704 INFO:teuthology.orchestra.run.trial012.stdout:Get:21 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-22T19:46:13.704 INFO:teuthology.orchestra.run.trial012.stdout:Get:22 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-22T19:46:13.704 INFO:teuthology.orchestra.run.trial012.stdout:Get:23 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-22T19:46:13.776 INFO:teuthology.orchestra.run.trial012.stdout:Get:24 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-22T19:46:13.778 INFO:teuthology.orchestra.run.trial012.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-22T19:46:13.778 INFO:teuthology.orchestra.run.trial012.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-22T19:46:13.849 INFO:teuthology.orchestra.run.trial012.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-22T19:46:13.849 INFO:teuthology.orchestra.run.trial012.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-22T19:46:13.849 INFO:teuthology.orchestra.run.trial012.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-22T19:46:13.851 INFO:teuthology.orchestra.run.trial012.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-22T19:46:13.851 INFO:teuthology.orchestra.run.trial012.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2026-02-22T19:46:13.851 INFO:teuthology.orchestra.run.trial012.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2026-02-22T19:46:13.858 INFO:teuthology.orchestra.run.trial012.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2026-02-22T19:46:13.904 INFO:teuthology.orchestra.run.trial098.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-22T19:46:13.917 INFO:teuthology.orchestra.run.trial098.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-22T19:46:13.925 INFO:teuthology.orchestra.run.trial012.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7320 B] 2026-02-22T19:46:13.925 INFO:teuthology.orchestra.run.trial012.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-22T19:46:13.930 INFO:teuthology.orchestra.run.trial012.stdout:Get:36 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-waitress all 1.4.4-1.1ubuntu1.1 [47.0 kB] 2026-02-22T19:46:13.950 INFO:teuthology.orchestra.run.trial098.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-22T19:46:13.959 INFO:teuthology.orchestra.run.trial098.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-22T19:46:13.962 INFO:teuthology.orchestra.run.trial098.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-22T19:46:13.962 INFO:teuthology.orchestra.run.trial098.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-22T19:46:13.964 INFO:teuthology.orchestra.run.trial098.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-22T19:46:13.984 INFO:teuthology.orchestra.run.trial098.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB] 2026-02-22T19:46:13.988 INFO:teuthology.orchestra.run.trial098.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB] 2026-02-22T19:46:13.993 INFO:teuthology.orchestra.run.trial098.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB] 2026-02-22T19:46:13.997 INFO:teuthology.orchestra.run.trial012.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2026-02-22T19:46:13.997 INFO:teuthology.orchestra.run.trial012.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2026-02-22T19:46:14.002 INFO:teuthology.orchestra.run.trial012.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4892 B] 2026-02-22T19:46:14.003 INFO:teuthology.orchestra.run.trial012.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2026-02-22T19:46:14.003 INFO:teuthology.orchestra.run.trial012.stdout:Get:41 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2026-02-22T19:46:14.003 INFO:teuthology.orchestra.run.trial012.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2026-02-22T19:46:14.004 INFO:teuthology.orchestra.run.trial012.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.3 [181 kB] 2026-02-22T19:46:14.069 INFO:teuthology.orchestra.run.trial012.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-22T19:46:14.070 INFO:teuthology.orchestra.run.trial012.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-22T19:46:14.073 INFO:teuthology.orchestra.run.trial012.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-22T19:46:14.076 INFO:teuthology.orchestra.run.trial098.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5034 B] 2026-02-22T19:46:14.076 INFO:teuthology.orchestra.run.trial098.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB] 2026-02-22T19:46:14.078 INFO:teuthology.orchestra.run.trial098.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB] 2026-02-22T19:46:14.081 INFO:teuthology.orchestra.run.trial098.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB] 2026-02-22T19:46:14.083 INFO:teuthology.orchestra.run.trial098.stdout:Get:20 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-22T19:46:14.083 INFO:teuthology.orchestra.run.trial098.stdout:Get:21 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-22T19:46:14.084 INFO:teuthology.orchestra.run.trial098.stdout:Get:22 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-22T19:46:14.085 INFO:teuthology.orchestra.run.trial098.stdout:Get:23 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-22T19:46:14.085 INFO:teuthology.orchestra.run.trial098.stdout:Get:24 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-22T19:46:14.142 INFO:teuthology.orchestra.run.trial012.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-22T19:46:14.142 INFO:teuthology.orchestra.run.trial012.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-22T19:46:14.167 INFO:teuthology.orchestra.run.trial012.stdout:Get:49 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-22T19:46:14.168 INFO:teuthology.orchestra.run.trial012.stdout:Get:50 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-22T19:46:14.169 INFO:teuthology.orchestra.run.trial012.stdout:Get:51 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-22T19:46:14.170 INFO:teuthology.orchestra.run.trial098.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-22T19:46:14.171 INFO:teuthology.orchestra.run.trial098.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-22T19:46:14.171 INFO:teuthology.orchestra.run.trial098.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-22T19:46:14.171 INFO:teuthology.orchestra.run.trial098.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-22T19:46:14.201 INFO:teuthology.orchestra.run.trial012.stdout:Get:52 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.2.3-689-gf90f4eae-1jammy [3256 kB] 2026-02-22T19:46:14.227 INFO:teuthology.orchestra.run.trial012.stdout:Get:53 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-22T19:46:14.227 INFO:teuthology.orchestra.run.trial012.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-22T19:46:14.227 INFO:teuthology.orchestra.run.trial012.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-22T19:46:14.228 INFO:teuthology.orchestra.run.trial012.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-22T19:46:14.228 INFO:teuthology.orchestra.run.trial012.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-22T19:46:14.258 INFO:teuthology.orchestra.run.trial098.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-22T19:46:14.261 INFO:teuthology.orchestra.run.trial098.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-22T19:46:14.261 INFO:teuthology.orchestra.run.trial098.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2026-02-22T19:46:14.261 INFO:teuthology.orchestra.run.trial098.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2026-02-22T19:46:14.262 INFO:teuthology.orchestra.run.trial098.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2026-02-22T19:46:14.262 INFO:teuthology.orchestra.run.trial098.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7320 B] 2026-02-22T19:46:14.286 INFO:teuthology.orchestra.run.trial012.stdout:Get:58 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-22T19:46:14.289 INFO:teuthology.orchestra.run.trial012.stdout:Get:59 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-22T19:46:14.290 INFO:teuthology.orchestra.run.trial012.stdout:Get:60 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-22T19:46:14.291 INFO:teuthology.orchestra.run.trial012.stdout:Get:61 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-22T19:46:14.346 INFO:teuthology.orchestra.run.trial098.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-22T19:46:14.347 INFO:teuthology.orchestra.run.trial098.stdout:Get:36 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-waitress all 1.4.4-1.1ubuntu1.1 [47.0 kB] 2026-02-22T19:46:14.347 INFO:teuthology.orchestra.run.trial098.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2026-02-22T19:46:14.348 INFO:teuthology.orchestra.run.trial098.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2026-02-22T19:46:14.358 INFO:teuthology.orchestra.run.trial012.stdout:Get:62 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-22T19:46:14.361 INFO:teuthology.orchestra.run.trial012.stdout:Get:63 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-22T19:46:14.363 INFO:teuthology.orchestra.run.trial012.stdout:Get:64 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB] 2026-02-22T19:46:14.364 INFO:teuthology.orchestra.run.trial012.stdout:Get:65 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB] 2026-02-22T19:46:14.364 INFO:teuthology.orchestra.run.trial012.stdout:Get:66 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-22T19:46:14.368 INFO:teuthology.orchestra.run.trial012.stdout:Get:67 https://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB] 2026-02-22T19:46:14.431 INFO:teuthology.orchestra.run.trial012.stdout:Get:68 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-22T19:46:14.433 INFO:teuthology.orchestra.run.trial012.stdout:Get:69 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-22T19:46:14.434 INFO:teuthology.orchestra.run.trial012.stdout:Get:70 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2026-02-22T19:46:14.434 INFO:teuthology.orchestra.run.trial098.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4892 B] 2026-02-22T19:46:14.434 INFO:teuthology.orchestra.run.trial098.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2026-02-22T19:46:14.434 INFO:teuthology.orchestra.run.trial098.stdout:Get:41 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2026-02-22T19:46:14.434 INFO:teuthology.orchestra.run.trial012.stdout:Get:71 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-22T19:46:14.434 INFO:teuthology.orchestra.run.trial098.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2026-02-22T19:46:14.435 INFO:teuthology.orchestra.run.trial098.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.3 [181 kB] 2026-02-22T19:46:14.438 INFO:teuthology.orchestra.run.trial098.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-22T19:46:14.503 INFO:teuthology.orchestra.run.trial012.stdout:Get:72 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-22T19:46:14.504 INFO:teuthology.orchestra.run.trial012.stdout:Get:73 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-22T19:46:14.504 INFO:teuthology.orchestra.run.trial012.stdout:Get:74 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-22T19:46:14.511 INFO:teuthology.orchestra.run.trial012.stdout:Get:75 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2026-02-22T19:46:14.511 INFO:teuthology.orchestra.run.trial012.stdout:Get:76 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-22T19:46:14.511 INFO:teuthology.orchestra.run.trial012.stdout:Get:77 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-22T19:46:14.521 INFO:teuthology.orchestra.run.trial098.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-22T19:46:14.524 INFO:teuthology.orchestra.run.trial098.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-22T19:46:14.524 INFO:teuthology.orchestra.run.trial098.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-22T19:46:14.525 INFO:teuthology.orchestra.run.trial098.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-22T19:46:14.575 INFO:teuthology.orchestra.run.trial012.stdout:Get:78 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-22T19:46:14.575 INFO:teuthology.orchestra.run.trial012.stdout:Get:79 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-22T19:46:14.609 INFO:teuthology.orchestra.run.trial098.stdout:Get:49 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-22T19:46:14.612 INFO:teuthology.orchestra.run.trial098.stdout:Get:50 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-22T19:46:14.612 INFO:teuthology.orchestra.run.trial098.stdout:Get:51 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-22T19:46:14.636 INFO:teuthology.orchestra.run.trial098.stdout:Get:52 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-22T19:46:14.637 INFO:teuthology.orchestra.run.trial098.stdout:Get:53 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-22T19:46:14.637 INFO:teuthology.orchestra.run.trial098.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-22T19:46:14.697 INFO:teuthology.orchestra.run.trial098.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-22T19:46:14.697 INFO:teuthology.orchestra.run.trial098.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-22T19:46:14.697 INFO:teuthology.orchestra.run.trial098.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-22T19:46:14.702 INFO:teuthology.orchestra.run.trial098.stdout:Get:58 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-22T19:46:14.785 INFO:teuthology.orchestra.run.trial098.stdout:Get:59 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-22T19:46:14.786 INFO:teuthology.orchestra.run.trial098.stdout:Get:60 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-22T19:46:14.787 INFO:teuthology.orchestra.run.trial098.stdout:Get:61 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-22T19:46:14.791 INFO:teuthology.orchestra.run.trial098.stdout:Get:62 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-22T19:46:14.795 INFO:teuthology.orchestra.run.trial098.stdout:Get:63 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB] 2026-02-22T19:46:14.795 INFO:teuthology.orchestra.run.trial098.stdout:Get:64 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB] 2026-02-22T19:46:14.873 INFO:teuthology.orchestra.run.trial098.stdout:Get:65 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-22T19:46:14.878 INFO:teuthology.orchestra.run.trial098.stdout:Get:66 https://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB] 2026-02-22T19:46:14.879 INFO:teuthology.orchestra.run.trial098.stdout:Get:67 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-22T19:46:14.882 INFO:teuthology.orchestra.run.trial098.stdout:Get:68 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-22T19:46:14.960 INFO:teuthology.orchestra.run.trial098.stdout:Get:69 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2026-02-22T19:46:14.961 INFO:teuthology.orchestra.run.trial098.stdout:Get:70 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-22T19:46:14.961 INFO:teuthology.orchestra.run.trial098.stdout:Get:71 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-22T19:46:14.962 INFO:teuthology.orchestra.run.trial098.stdout:Get:72 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-22T19:46:14.964 INFO:teuthology.orchestra.run.trial098.stdout:Get:73 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-22T19:46:14.973 INFO:teuthology.orchestra.run.trial098.stdout:Get:74 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2026-02-22T19:46:15.048 INFO:teuthology.orchestra.run.trial098.stdout:Get:75 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-22T19:46:15.048 INFO:teuthology.orchestra.run.trial098.stdout:Get:76 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-22T19:46:15.050 INFO:teuthology.orchestra.run.trial098.stdout:Get:77 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-22T19:46:15.051 INFO:teuthology.orchestra.run.trial098.stdout:Get:78 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-22T19:46:15.552 INFO:teuthology.orchestra.run.trial098.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.2.3-689-gf90f4eae-1jammy [3256 kB] 2026-02-22T19:46:15.891 INFO:teuthology.orchestra.run.trial012.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.2.3-689-gf90f4eae-1jammy [979 kB] 2026-02-22T19:46:16.556 INFO:teuthology.orchestra.run.trial012.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.2.3-689-gf90f4eae-1jammy [357 kB] 2026-02-22T19:46:16.807 INFO:teuthology.orchestra.run.trial012.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.2.3-689-gf90f4eae-1jammy [32.9 kB] 2026-02-22T19:46:16.817 INFO:teuthology.orchestra.run.trial012.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.2.3-689-gf90f4eae-1jammy [184 kB] 2026-02-22T19:46:16.947 INFO:teuthology.orchestra.run.trial012.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.2.3-689-gf90f4eae-1jammy [70.0 kB] 2026-02-22T19:46:16.960 INFO:teuthology.orchestra.run.trial012.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.2.3-689-gf90f4eae-1jammy [335 kB] 2026-02-22T19:46:17.206 INFO:teuthology.orchestra.run.trial012.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.2.3-689-gf90f4eae-1jammy [6935 kB] 2026-02-22T19:46:17.790 INFO:teuthology.orchestra.run.trial098.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.2.3-689-gf90f4eae-1jammy [979 kB] 2026-02-22T19:46:18.512 INFO:teuthology.orchestra.run.trial098.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.2.3-689-gf90f4eae-1jammy [357 kB] 2026-02-22T19:46:18.742 INFO:teuthology.orchestra.run.trial098.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.2.3-689-gf90f4eae-1jammy [32.9 kB] 2026-02-22T19:46:18.782 INFO:teuthology.orchestra.run.trial098.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.2.3-689-gf90f4eae-1jammy [184 kB] 2026-02-22T19:46:18.914 INFO:teuthology.orchestra.run.trial098.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.2.3-689-gf90f4eae-1jammy [70.0 kB] 2026-02-22T19:46:18.941 INFO:teuthology.orchestra.run.trial098.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.2.3-689-gf90f4eae-1jammy [335 kB] 2026-02-22T19:46:19.187 INFO:teuthology.orchestra.run.trial098.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.2.3-689-gf90f4eae-1jammy [6935 kB] 2026-02-22T19:46:21.956 INFO:teuthology.orchestra.run.trial012.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.2.3-689-gf90f4eae-1jammy [112 kB] 2026-02-22T19:46:22.045 INFO:teuthology.orchestra.run.trial012.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.2.3-689-gf90f4eae-1jammy [470 kB] 2026-02-22T19:46:22.484 INFO:teuthology.orchestra.run.trial012.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.2.3-689-gf90f4eae-1jammy [26.5 MB] 2026-02-22T19:46:24.517 INFO:teuthology.orchestra.run.trial098.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.2.3-689-gf90f4eae-1jammy [112 kB] 2026-02-22T19:46:24.589 INFO:teuthology.orchestra.run.trial098.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.2.3-689-gf90f4eae-1jammy [470 kB] 2026-02-22T19:46:24.933 INFO:teuthology.orchestra.run.trial098.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.2.3-689-gf90f4eae-1jammy [26.5 MB] 2026-02-22T19:46:45.432 INFO:teuthology.orchestra.run.trial012.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.2.3-689-gf90f4eae-1jammy [5178 kB] 2026-02-22T19:46:47.414 INFO:teuthology.orchestra.run.trial098.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.2.3-689-gf90f4eae-1jammy [5178 kB] 2026-02-22T19:46:50.400 INFO:teuthology.orchestra.run.trial012.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.2.3-689-gf90f4eae-1jammy [248 kB] 2026-02-22T19:46:50.544 INFO:teuthology.orchestra.run.trial098.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.2.3-689-gf90f4eae-1jammy [248 kB] 2026-02-22T19:46:50.594 INFO:teuthology.orchestra.run.trial012.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.2.3-689-gf90f4eae-1jammy [125 kB] 2026-02-22T19:46:50.689 INFO:teuthology.orchestra.run.trial012.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.2.3-689-gf90f4eae-1jammy [1081 kB] 2026-02-22T19:46:50.793 INFO:teuthology.orchestra.run.trial098.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.2.3-689-gf90f4eae-1jammy [125 kB] 2026-02-22T19:46:50.819 INFO:teuthology.orchestra.run.trial098.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.2.3-689-gf90f4eae-1jammy [1081 kB] 2026-02-22T19:46:51.518 INFO:teuthology.orchestra.run.trial012.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.2.3-689-gf90f4eae-1jammy [6239 kB] 2026-02-22T19:46:51.757 INFO:teuthology.orchestra.run.trial098.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.2.3-689-gf90f4eae-1jammy [6239 kB] 2026-02-22T19:46:56.785 INFO:teuthology.orchestra.run.trial012.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.2.3-689-gf90f4eae-1jammy [23.0 MB] 2026-02-22T19:46:57.475 INFO:teuthology.orchestra.run.trial098.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.2.3-689-gf90f4eae-1jammy [23.0 MB] 2026-02-22T19:47:26.095 INFO:teuthology.orchestra.run.trial012.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.2.3-689-gf90f4eae-1jammy [14.2 kB] 2026-02-22T19:47:26.098 INFO:teuthology.orchestra.run.trial012.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.2.3-689-gf90f4eae-1jammy [1173 kB] 2026-02-22T19:47:27.349 INFO:teuthology.orchestra.run.trial012.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.2.3-689-gf90f4eae-1jammy [2503 kB] 2026-02-22T19:47:27.561 INFO:teuthology.orchestra.run.trial098.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.2.3-689-gf90f4eae-1jammy [14.2 kB] 2026-02-22T19:47:27.598 INFO:teuthology.orchestra.run.trial098.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.2.3-689-gf90f4eae-1jammy [1173 kB] 2026-02-22T19:47:29.744 INFO:teuthology.orchestra.run.trial098.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.2.3-689-gf90f4eae-1jammy [2503 kB] 2026-02-22T19:47:29.823 INFO:teuthology.orchestra.run.trial012.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.2.3-689-gf90f4eae-1jammy [798 kB] 2026-02-22T19:47:30.545 INFO:teuthology.orchestra.run.trial012.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.2.3-689-gf90f4eae-1jammy [157 kB] 2026-02-22T19:47:30.680 INFO:teuthology.orchestra.run.trial012.stdout:Get:101 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.2.3-689-gf90f4eae-1jammy [2396 kB] 2026-02-22T19:47:33.178 INFO:teuthology.orchestra.run.trial012.stdout:Get:102 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.2.3-689-gf90f4eae-1jammy [8625 kB] 2026-02-22T19:47:33.936 INFO:teuthology.orchestra.run.trial098.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.2.3-689-gf90f4eae-1jammy [798 kB] 2026-02-22T19:47:34.877 INFO:teuthology.orchestra.run.trial098.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.2.3-689-gf90f4eae-1jammy [157 kB] 2026-02-22T19:47:35.046 INFO:teuthology.orchestra.run.trial098.stdout:Get:101 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.2.3-689-gf90f4eae-1jammy [2396 kB] 2026-02-22T19:47:37.327 INFO:teuthology.orchestra.run.trial098.stdout:Get:102 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.2.3-689-gf90f4eae-1jammy [8625 kB] 2026-02-22T19:47:47.798 INFO:teuthology.orchestra.run.trial012.stdout:Get:103 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.2.3-689-gf90f4eae-1jammy [14.3 kB] 2026-02-22T19:47:47.829 INFO:teuthology.orchestra.run.trial012.stdout:Get:104 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.2.3-689-gf90f4eae-1jammy [52.1 MB] 2026-02-22T19:47:50.382 INFO:teuthology.orchestra.run.trial098.stdout:Get:103 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.2.3-689-gf90f4eae-1jammy [14.3 kB] 2026-02-22T19:47:50.409 INFO:teuthology.orchestra.run.trial098.stdout:Get:104 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.2.3-689-gf90f4eae-1jammy [52.1 MB] 2026-02-22T19:49:08.224 INFO:teuthology.orchestra.run.trial012.stdout:Get:105 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.2.3-689-gf90f4eae-1jammy [135 kB] 2026-02-22T19:49:08.387 INFO:teuthology.orchestra.run.trial012.stdout:Get:106 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.2.3-689-gf90f4eae-1jammy [41.0 kB] 2026-02-22T19:49:08.432 INFO:teuthology.orchestra.run.trial012.stdout:Get:107 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.2.3-689-gf90f4eae-1jammy [13.7 MB] 2026-02-22T19:49:13.109 INFO:teuthology.orchestra.run.trial098.stdout:Get:105 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.2.3-689-gf90f4eae-1jammy [135 kB] 2026-02-22T19:49:13.322 INFO:teuthology.orchestra.run.trial098.stdout:Get:106 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.2.3-689-gf90f4eae-1jammy [41.0 kB] 2026-02-22T19:49:13.377 INFO:teuthology.orchestra.run.trial098.stdout:Get:107 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.2.3-689-gf90f4eae-1jammy [13.7 MB] 2026-02-22T19:49:24.893 INFO:teuthology.orchestra.run.trial012.stdout:Get:108 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.2.3-689-gf90f4eae-1jammy [92.3 kB] 2026-02-22T19:49:24.965 INFO:teuthology.orchestra.run.trial012.stdout:Get:109 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.2.3-689-gf90f4eae-1jammy [3151 kB] 2026-02-22T19:49:25.512 INFO:teuthology.orchestra.run.trial098.stdout:Get:108 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.2.3-689-gf90f4eae-1jammy [92.3 kB] 2026-02-22T19:49:25.557 INFO:teuthology.orchestra.run.trial098.stdout:Get:109 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.2.3-689-gf90f4eae-1jammy [3151 kB] 2026-02-22T19:49:27.628 INFO:teuthology.orchestra.run.trial098.stdout:Fetched 181 MB in 3min 15s (929 kB/s) 2026-02-22T19:49:27.644 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-22T19:49:27.800 INFO:teuthology.orchestra.run.trial098.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167442 files and directories currently installed.) 2026-02-22T19:49:27.802 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-22T19:49:27.803 INFO:teuthology.orchestra.run.trial012.stdout:Fetched 181 MB in 3min 15s (927 kB/s) 2026-02-22T19:49:27.804 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-22T19:49:27.819 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-22T19:49:27.825 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-22T19:49:27.838 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-22T19:49:27.838 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-22T19:49:27.854 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-22T19:49:27.866 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-22T19:49:27.866 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-22T19:49:27.885 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-22T19:49:27.894 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T19:49:27.898 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:27.929 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-22T19:49:27.937 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T19:49:27.937 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:27.953 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-22T19:49:27.961 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T19:49:27.962 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:27.973 INFO:teuthology.orchestra.run.trial012.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167442 files and directories currently installed.) 2026-02-22T19:49:27.975 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-22T19:49:27.977 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-22T19:49:27.981 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-22T19:49:27.989 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-22T19:49:27.989 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-22T19:49:27.997 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-22T19:49:28.006 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-22T19:49:28.006 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package librados2. 2026-02-22T19:49:28.006 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-22T19:49:28.014 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../007-librados2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.014 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking librados2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.022 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-22T19:49:28.031 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-22T19:49:28.031 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-22T19:49:28.049 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-22T19:49:28.055 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libnbd0. 2026-02-22T19:49:28.057 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T19:49:28.062 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:28.063 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../008-libnbd0_1.10.5-1_amd64.deb ... 2026-02-22T19:49:28.064 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-22T19:49:28.079 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package librbd1. 2026-02-22T19:49:28.087 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../009-librbd1_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.087 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking librbd1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.092 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-22T19:49:28.100 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T19:49:28.100 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:28.116 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-22T19:49:28.124 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T19:49:28.124 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:28.128 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libcephfs2. 2026-02-22T19:49:28.137 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../010-libcephfs2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.137 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libcephfs2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.144 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-22T19:49:28.152 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-22T19:49:28.153 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-22T19:49:28.158 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-rados. 2026-02-22T19:49:28.166 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../011-python3-rados_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.166 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-rados (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.169 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package librados2. 2026-02-22T19:49:28.177 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../007-librados2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.178 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking librados2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.182 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-22T19:49:28.191 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../012-python3-ceph-argparse_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:28.191 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-ceph-argparse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.205 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-cephfs. 2026-02-22T19:49:28.213 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../013-python3-cephfs_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.213 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-cephfs (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.218 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libnbd0. 2026-02-22T19:49:28.227 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../008-libnbd0_1.10.5-1_amd64.deb ... 2026-02-22T19:49:28.227 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-22T19:49:28.229 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-22T19:49:28.236 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../014-python3-ceph-common_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:28.237 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.241 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package librbd1. 2026-02-22T19:49:28.249 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../009-librbd1_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.249 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking librbd1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.254 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-22T19:49:28.262 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-22T19:49:28.262 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-22T19:49:28.278 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-prettytable. 2026-02-22T19:49:28.286 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2026-02-22T19:49:28.286 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-22T19:49:28.290 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libcephfs2. 2026-02-22T19:49:28.298 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../010-libcephfs2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.299 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libcephfs2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.300 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-rbd. 2026-02-22T19:49:28.308 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../017-python3-rbd_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.308 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-rbd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.320 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-rados. 2026-02-22T19:49:28.326 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-22T19:49:28.328 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../011-python3-rados_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.328 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-rados (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.334 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-22T19:49:28.334 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-22T19:49:28.345 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-22T19:49:28.352 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libreadline-dev:amd64. 2026-02-22T19:49:28.353 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../012-python3-ceph-argparse_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:28.353 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-ceph-argparse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.360 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../019-libreadline-dev_8.1.2-1_amd64.deb ... 2026-02-22T19:49:28.361 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ... 2026-02-22T19:49:28.367 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-cephfs. 2026-02-22T19:49:28.375 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../013-python3-cephfs_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.375 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-cephfs (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.378 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package liblua5.3-dev:amd64. 2026-02-22T19:49:28.386 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../020-liblua5.3-dev_5.3.6-1build1_amd64.deb ... 2026-02-22T19:49:28.387 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ... 2026-02-22T19:49:28.390 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-22T19:49:28.399 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../014-python3-ceph-common_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:28.399 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.404 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package lua5.1. 2026-02-22T19:49:28.412 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../021-lua5.1_5.1.5-8.1build4_amd64.deb ... 2026-02-22T19:49:28.412 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ... 2026-02-22T19:49:28.416 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-22T19:49:28.424 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-22T19:49:28.424 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-22T19:49:28.432 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package lua-any. 2026-02-22T19:49:28.440 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../022-lua-any_27ubuntu1_all.deb ... 2026-02-22T19:49:28.441 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-prettytable. 2026-02-22T19:49:28.441 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking lua-any (27ubuntu1) ... 2026-02-22T19:49:28.449 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2026-02-22T19:49:28.449 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-22T19:49:28.455 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package zip. 2026-02-22T19:49:28.463 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../023-zip_3.0-12build2_amd64.deb ... 2026-02-22T19:49:28.463 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking zip (3.0-12build2) ... 2026-02-22T19:49:28.464 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-rbd. 2026-02-22T19:49:28.471 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../017-python3-rbd_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.472 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-rbd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.481 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package unzip. 2026-02-22T19:49:28.488 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-22T19:49:28.489 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../024-unzip_6.0-26ubuntu3.2_amd64.deb ... 2026-02-22T19:49:28.489 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking unzip (6.0-26ubuntu3.2) ... 2026-02-22T19:49:28.497 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-22T19:49:28.497 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-22T19:49:28.506 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package luarocks. 2026-02-22T19:49:28.514 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libreadline-dev:amd64. 2026-02-22T19:49:28.514 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../025-luarocks_3.8.0+dfsg1-1_all.deb ... 2026-02-22T19:49:28.515 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ... 2026-02-22T19:49:28.522 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../019-libreadline-dev_8.1.2-1_amd64.deb ... 2026-02-22T19:49:28.522 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ... 2026-02-22T19:49:28.539 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package liblua5.3-dev:amd64. 2026-02-22T19:49:28.547 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../020-liblua5.3-dev_5.3.6-1build1_amd64.deb ... 2026-02-22T19:49:28.547 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ... 2026-02-22T19:49:28.553 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package librgw2. 2026-02-22T19:49:28.561 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../026-librgw2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.561 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking librgw2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.564 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package lua5.1. 2026-02-22T19:49:28.572 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../021-lua5.1_5.1.5-8.1build4_amd64.deb ... 2026-02-22T19:49:28.573 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ... 2026-02-22T19:49:28.592 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package lua-any. 2026-02-22T19:49:28.599 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../022-lua-any_27ubuntu1_all.deb ... 2026-02-22T19:49:28.599 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking lua-any (27ubuntu1) ... 2026-02-22T19:49:28.613 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package zip. 2026-02-22T19:49:28.621 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../023-zip_3.0-12build2_amd64.deb ... 2026-02-22T19:49:28.621 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking zip (3.0-12build2) ... 2026-02-22T19:49:28.634 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-rgw. 2026-02-22T19:49:28.637 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package unzip. 2026-02-22T19:49:28.643 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../027-python3-rgw_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.643 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-rgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.645 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../024-unzip_6.0-26ubuntu3.2_amd64.deb ... 2026-02-22T19:49:28.646 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking unzip (6.0-26ubuntu3.2) ... 2026-02-22T19:49:28.658 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-22T19:49:28.663 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package luarocks. 2026-02-22T19:49:28.666 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../028-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-22T19:49:28.666 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-22T19:49:28.671 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../025-luarocks_3.8.0+dfsg1-1_all.deb ... 2026-02-22T19:49:28.672 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ... 2026-02-22T19:49:28.681 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libradosstriper1. 2026-02-22T19:49:28.689 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../029-libradosstriper1_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.689 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libradosstriper1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.707 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-common. 2026-02-22T19:49:28.709 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package librgw2. 2026-02-22T19:49:28.715 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../030-ceph-common_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.716 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.717 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../026-librgw2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.717 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking librgw2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.789 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-rgw. 2026-02-22T19:49:28.797 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../027-python3-rgw_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.798 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-rgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.813 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-22T19:49:28.821 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../028-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-22T19:49:28.821 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-22T19:49:28.836 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libradosstriper1. 2026-02-22T19:49:28.844 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../029-libradosstriper1_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.845 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libradosstriper1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.863 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-common. 2026-02-22T19:49:28.871 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../030-ceph-common_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.871 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.920 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-base. 2026-02-22T19:49:28.928 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../031-ceph-base_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:28.933 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-base (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:28.997 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-22T19:49:29.005 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../032-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-22T19:49:29.006 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-22T19:49:29.020 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-cheroot. 2026-02-22T19:49:29.028 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../033-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-22T19:49:29.029 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-22T19:49:29.046 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-22T19:49:29.054 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../034-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-22T19:49:29.054 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-22T19:49:29.067 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-22T19:49:29.075 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-base. 2026-02-22T19:49:29.075 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../035-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-22T19:49:29.076 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-22T19:49:29.083 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../031-ceph-base_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.088 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-base (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.090 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-22T19:49:29.099 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../036-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-22T19:49:29.099 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-22T19:49:29.114 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-tempora. 2026-02-22T19:49:29.122 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../037-python3-tempora_4.1.2-1_all.deb ... 2026-02-22T19:49:29.123 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-22T19:49:29.138 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-portend. 2026-02-22T19:49:29.146 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../038-python3-portend_3.0.0-1_all.deb ... 2026-02-22T19:49:29.146 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-22T19:49:29.153 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-22T19:49:29.160 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-22T19:49:29.161 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../032-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-22T19:49:29.161 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-22T19:49:29.169 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../039-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-22T19:49:29.169 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-22T19:49:29.176 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-cheroot. 2026-02-22T19:49:29.184 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../033-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-22T19:49:29.185 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-22T19:49:29.185 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-22T19:49:29.192 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../040-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-22T19:49:29.193 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-22T19:49:29.202 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-22T19:49:29.210 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../034-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-22T19:49:29.211 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-22T19:49:29.216 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-natsort. 2026-02-22T19:49:29.224 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../041-python3-natsort_8.0.2-1_all.deb ... 2026-02-22T19:49:29.225 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-22T19:49:29.225 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-22T19:49:29.233 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../035-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-22T19:49:29.233 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-22T19:49:29.241 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-logutils. 2026-02-22T19:49:29.248 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-22T19:49:29.249 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../042-python3-logutils_0.3.3-8_all.deb ... 2026-02-22T19:49:29.249 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-logutils (0.3.3-8) ... 2026-02-22T19:49:29.256 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../036-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-22T19:49:29.256 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-22T19:49:29.264 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-mako. 2026-02-22T19:49:29.270 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-tempora. 2026-02-22T19:49:29.272 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../043-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2026-02-22T19:49:29.273 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2026-02-22T19:49:29.278 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../037-python3-tempora_4.1.2-1_all.deb ... 2026-02-22T19:49:29.279 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-22T19:49:29.290 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-simplegeneric. 2026-02-22T19:49:29.294 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-portend. 2026-02-22T19:49:29.299 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../044-python3-simplegeneric_0.8.1-3_all.deb ... 2026-02-22T19:49:29.299 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2026-02-22T19:49:29.302 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../038-python3-portend_3.0.0-1_all.deb ... 2026-02-22T19:49:29.303 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-22T19:49:29.314 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-singledispatch. 2026-02-22T19:49:29.317 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-22T19:49:29.323 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../045-python3-singledispatch_3.4.0.3-3_all.deb ... 2026-02-22T19:49:29.323 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2026-02-22T19:49:29.325 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../039-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-22T19:49:29.326 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-22T19:49:29.337 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-webob. 2026-02-22T19:49:29.340 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-22T19:49:29.345 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../046-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-22T19:49:29.346 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-22T19:49:29.348 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../040-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-22T19:49:29.349 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-22T19:49:29.362 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-waitress. 2026-02-22T19:49:29.370 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../047-python3-waitress_1.4.4-1.1ubuntu1.1_all.deb ... 2026-02-22T19:49:29.372 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-natsort. 2026-02-22T19:49:29.372 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1.1) ... 2026-02-22T19:49:29.380 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../041-python3-natsort_8.0.2-1_all.deb ... 2026-02-22T19:49:29.380 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-22T19:49:29.387 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-tempita. 2026-02-22T19:49:29.395 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-logutils. 2026-02-22T19:49:29.396 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../048-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2026-02-22T19:49:29.396 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2026-02-22T19:49:29.403 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../042-python3-logutils_0.3.3-8_all.deb ... 2026-02-22T19:49:29.404 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-logutils (0.3.3-8) ... 2026-02-22T19:49:29.411 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-paste. 2026-02-22T19:49:29.418 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-mako. 2026-02-22T19:49:29.419 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../049-python3-paste_3.5.0+dfsg1-1_all.deb ... 2026-02-22T19:49:29.419 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2026-02-22T19:49:29.426 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../043-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2026-02-22T19:49:29.426 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2026-02-22T19:49:29.444 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-simplegeneric. 2026-02-22T19:49:29.447 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2026-02-22T19:49:29.452 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../044-python3-simplegeneric_0.8.1-3_all.deb ... 2026-02-22T19:49:29.452 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2026-02-22T19:49:29.456 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../050-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2026-02-22T19:49:29.456 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2026-02-22T19:49:29.466 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-singledispatch. 2026-02-22T19:49:29.470 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-pastedeploy. 2026-02-22T19:49:29.474 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../045-python3-singledispatch_3.4.0.3-3_all.deb ... 2026-02-22T19:49:29.474 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2026-02-22T19:49:29.478 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../051-python3-pastedeploy_2.1.1-1_all.deb ... 2026-02-22T19:49:29.478 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2026-02-22T19:49:29.489 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-webob. 2026-02-22T19:49:29.493 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-webtest. 2026-02-22T19:49:29.497 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../046-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-22T19:49:29.498 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-22T19:49:29.501 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../052-python3-webtest_2.0.35-1_all.deb ... 2026-02-22T19:49:29.501 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-webtest (2.0.35-1) ... 2026-02-22T19:49:29.514 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-waitress. 2026-02-22T19:49:29.516 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-pecan. 2026-02-22T19:49:29.523 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../047-python3-waitress_1.4.4-1.1ubuntu1.1_all.deb ... 2026-02-22T19:49:29.525 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../053-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2026-02-22T19:49:29.525 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1.1) ... 2026-02-22T19:49:29.525 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2026-02-22T19:49:29.540 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-tempita. 2026-02-22T19:49:29.548 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../048-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2026-02-22T19:49:29.549 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2026-02-22T19:49:29.550 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-werkzeug. 2026-02-22T19:49:29.558 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../054-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.3_all.deb ... 2026-02-22T19:49:29.558 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.3) ... 2026-02-22T19:49:29.563 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-paste. 2026-02-22T19:49:29.571 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../049-python3-paste_3.5.0+dfsg1-1_all.deb ... 2026-02-22T19:49:29.572 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2026-02-22T19:49:29.578 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-22T19:49:29.586 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../055-ceph-mgr-modules-core_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:29.586 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-mgr-modules-core (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.599 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2026-02-22T19:49:29.607 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../050-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2026-02-22T19:49:29.608 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2026-02-22T19:49:29.614 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-22T19:49:29.622 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-pastedeploy. 2026-02-22T19:49:29.623 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.623 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libsqlite3-mod-ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.630 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../051-python3-pastedeploy_2.1.1-1_all.deb ... 2026-02-22T19:49:29.631 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2026-02-22T19:49:29.638 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-mgr. 2026-02-22T19:49:29.646 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-webtest. 2026-02-22T19:49:29.647 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../057-ceph-mgr_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.647 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-mgr (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.654 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../052-python3-webtest_2.0.35-1_all.deb ... 2026-02-22T19:49:29.655 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-webtest (2.0.35-1) ... 2026-02-22T19:49:29.669 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-mon. 2026-02-22T19:49:29.670 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-pecan. 2026-02-22T19:49:29.677 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../058-ceph-mon_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.678 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-mon (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.678 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../053-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2026-02-22T19:49:29.679 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2026-02-22T19:49:29.703 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-werkzeug. 2026-02-22T19:49:29.711 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../054-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.3_all.deb ... 2026-02-22T19:49:29.712 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.3) ... 2026-02-22T19:49:29.731 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-22T19:49:29.736 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-22T19:49:29.739 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../055-ceph-mgr-modules-core_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:29.740 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-mgr-modules-core (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.744 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../059-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-22T19:49:29.745 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-22T19:49:29.761 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-osd. 2026-02-22T19:49:29.768 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-22T19:49:29.769 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../060-ceph-osd_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.769 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-osd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.776 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.777 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libsqlite3-mod-ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.792 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-mgr. 2026-02-22T19:49:29.800 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../057-ceph-mgr_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.801 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-mgr (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.823 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-mon. 2026-02-22T19:49:29.832 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../058-ceph-mon_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.832 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-mon (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.890 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-22T19:49:29.899 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../059-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-22T19:49:29.899 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-22T19:49:29.915 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-osd. 2026-02-22T19:49:29.923 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../060-ceph-osd_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.923 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-osd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.924 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph. 2026-02-22T19:49:29.932 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../061-ceph_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.933 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.946 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-fuse. 2026-02-22T19:49:29.954 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../062-ceph-fuse_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.955 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:29.979 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-mds. 2026-02-22T19:49:29.987 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../063-ceph-mds_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:29.988 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-mds (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.021 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package cephadm. 2026-02-22T19:49:30.029 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../064-cephadm_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:30.030 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.046 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-22T19:49:30.054 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../065-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-22T19:49:30.054 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-22T19:49:30.075 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-22T19:49:30.083 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../066-ceph-mgr-cephadm_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:30.084 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-mgr-cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.085 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph. 2026-02-22T19:49:30.093 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../061-ceph_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:30.094 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.103 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-22T19:49:30.108 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-fuse. 2026-02-22T19:49:30.112 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../067-python3-repoze.lru_0.7-2_all.deb ... 2026-02-22T19:49:30.112 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-22T19:49:30.116 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../062-ceph-fuse_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:30.117 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.128 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-routes. 2026-02-22T19:49:30.136 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../068-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-22T19:49:30.136 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-22T19:49:30.140 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-mds. 2026-02-22T19:49:30.149 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../063-ceph-mds_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:30.149 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-mds (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.155 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-22T19:49:30.164 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../069-ceph-mgr-dashboard_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:30.164 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-mgr-dashboard (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.182 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package cephadm. 2026-02-22T19:49:30.191 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../064-cephadm_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:30.191 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.207 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-22T19:49:30.216 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../065-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-22T19:49:30.216 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-22T19:49:30.236 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-22T19:49:30.245 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../066-ceph-mgr-cephadm_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:30.245 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-mgr-cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.265 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-22T19:49:30.274 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../067-python3-repoze.lru_0.7-2_all.deb ... 2026-02-22T19:49:30.274 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-22T19:49:30.289 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-routes. 2026-02-22T19:49:30.298 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../068-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-22T19:49:30.298 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-22T19:49:30.318 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-22T19:49:30.326 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../069-ceph-mgr-dashboard_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:30.326 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-mgr-dashboard (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.368 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-22T19:49:30.376 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../070-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-22T19:49:30.377 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-22T19:49:30.414 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-joblib. 2026-02-22T19:49:30.422 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../071-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-22T19:49:30.423 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-22T19:49:30.449 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-22T19:49:30.458 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../072-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-22T19:49:30.458 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-22T19:49:30.473 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-sklearn. 2026-02-22T19:49:30.481 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../073-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-22T19:49:30.481 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-22T19:49:30.528 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-22T19:49:30.537 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../070-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-22T19:49:30.537 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-22T19:49:30.564 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-22T19:49:30.573 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../074-ceph-mgr-diskprediction-local_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:30.573 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-mgr-diskprediction-local (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.575 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-joblib. 2026-02-22T19:49:30.584 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../071-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-22T19:49:30.584 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-22T19:49:30.611 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-22T19:49:30.619 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../072-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-22T19:49:30.620 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-22T19:49:30.635 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-sklearn. 2026-02-22T19:49:30.643 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../073-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-22T19:49:30.643 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-22T19:49:30.724 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-22T19:49:30.733 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../074-ceph-mgr-diskprediction-local_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:30.733 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-mgr-diskprediction-local (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.741 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-cachetools. 2026-02-22T19:49:30.750 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../075-python3-cachetools_5.0.0-1_all.deb ... 2026-02-22T19:49:30.750 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-22T19:49:30.765 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-rsa. 2026-02-22T19:49:30.773 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../076-python3-rsa_4.8-1_all.deb ... 2026-02-22T19:49:30.774 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-22T19:49:30.790 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-google-auth. 2026-02-22T19:49:30.798 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../077-python3-google-auth_1.5.1-3_all.deb ... 2026-02-22T19:49:30.799 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-22T19:49:30.816 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-22T19:49:30.825 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../078-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-22T19:49:30.825 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-22T19:49:30.841 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-websocket. 2026-02-22T19:49:30.850 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../079-python3-websocket_1.2.3-1_all.deb ... 2026-02-22T19:49:30.850 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-22T19:49:30.867 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-22T19:49:30.876 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../080-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-22T19:49:30.876 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-22T19:49:30.903 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-cachetools. 2026-02-22T19:49:30.911 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../075-python3-cachetools_5.0.0-1_all.deb ... 2026-02-22T19:49:30.912 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-22T19:49:30.926 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-rsa. 2026-02-22T19:49:30.935 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../076-python3-rsa_4.8-1_all.deb ... 2026-02-22T19:49:30.935 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-22T19:49:30.952 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-google-auth. 2026-02-22T19:49:30.961 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../077-python3-google-auth_1.5.1-3_all.deb ... 2026-02-22T19:49:30.962 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-22T19:49:30.972 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-22T19:49:30.979 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-22T19:49:30.981 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../081-ceph-mgr-k8sevents_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:30.982 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-mgr-k8sevents (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:30.987 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../078-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-22T19:49:30.988 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-22T19:49:30.996 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-22T19:49:31.003 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-websocket. 2026-02-22T19:49:31.005 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../082-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-22T19:49:31.012 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../079-python3-websocket_1.2.3-1_all.deb ... 2026-02-22T19:49:31.012 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-22T19:49:31.012 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-22T19:49:31.028 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-22T19:49:31.029 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-22T19:49:31.036 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../083-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-22T19:49:31.036 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-22T19:49:31.038 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../080-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-22T19:49:31.038 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-22T19:49:31.051 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package jq. 2026-02-22T19:49:31.059 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../084-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-22T19:49:31.060 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-22T19:49:31.074 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package socat. 2026-02-22T19:49:31.083 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../085-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-22T19:49:31.083 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-22T19:49:31.102 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package xmlstarlet. 2026-02-22T19:49:31.111 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../086-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-22T19:49:31.112 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-22T19:49:31.134 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-22T19:49:31.143 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../081-ceph-mgr-k8sevents_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:31.143 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-mgr-k8sevents (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:31.148 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-test. 2026-02-22T19:49:31.157 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../087-ceph-test_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:31.157 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-22T19:49:31.158 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-test (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:31.166 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../082-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-22T19:49:31.172 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-22T19:49:31.188 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-22T19:49:31.196 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../083-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-22T19:49:31.197 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-22T19:49:31.211 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package jq. 2026-02-22T19:49:31.219 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../084-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-22T19:49:31.220 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-22T19:49:31.233 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package socat. 2026-02-22T19:49:31.241 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../085-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-22T19:49:31.241 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-22T19:49:31.260 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package xmlstarlet. 2026-02-22T19:49:31.269 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../086-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-22T19:49:31.269 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-22T19:49:31.306 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-test. 2026-02-22T19:49:31.314 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../087-ceph-test_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:31.315 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-test (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:31.576 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package ceph-volume. 2026-02-22T19:49:31.585 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../088-ceph-volume_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:31.585 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking ceph-volume (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:31.608 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-22T19:49:31.617 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../089-libcephfs-dev_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:31.617 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking libcephfs-dev (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:31.632 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package lua-socket:amd64. 2026-02-22T19:49:31.641 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../090-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ... 2026-02-22T19:49:31.642 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2026-02-22T19:49:31.663 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package lua-sec:amd64. 2026-02-22T19:49:31.671 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../091-lua-sec_1.0.2-1_amd64.deb ... 2026-02-22T19:49:31.672 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ... 2026-02-22T19:49:31.688 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package nvme-cli. 2026-02-22T19:49:31.697 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../092-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-22T19:49:31.697 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-22T19:49:31.729 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package pkg-config. 2026-02-22T19:49:31.738 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../093-pkg-config_0.29.2-1ubuntu3_amd64.deb ... 2026-02-22T19:49:31.738 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ... 2026-02-22T19:49:31.754 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-22T19:49:31.763 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../094-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-22T19:49:31.763 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-22T19:49:31.787 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package ceph-volume. 2026-02-22T19:49:31.792 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-22T19:49:31.796 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../088-ceph-volume_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T19:49:31.796 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking ceph-volume (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:31.801 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../095-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-22T19:49:31.801 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-22T19:49:31.816 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-pastescript. 2026-02-22T19:49:31.819 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-22T19:49:31.825 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../096-python3-pastescript_2.0.2-4_all.deb ... 2026-02-22T19:49:31.825 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2026-02-22T19:49:31.828 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../089-libcephfs-dev_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:31.828 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking libcephfs-dev (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:31.842 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-pluggy. 2026-02-22T19:49:31.844 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package lua-socket:amd64. 2026-02-22T19:49:31.851 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../097-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-22T19:49:31.851 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-22T19:49:31.852 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../090-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ... 2026-02-22T19:49:31.852 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2026-02-22T19:49:31.867 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-psutil. 2026-02-22T19:49:31.874 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package lua-sec:amd64. 2026-02-22T19:49:31.875 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../098-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-22T19:49:31.876 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-22T19:49:31.883 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../091-lua-sec_1.0.2-1_amd64.deb ... 2026-02-22T19:49:31.883 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ... 2026-02-22T19:49:31.894 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-py. 2026-02-22T19:49:31.900 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package nvme-cli. 2026-02-22T19:49:31.903 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../099-python3-py_1.10.0-1_all.deb ... 2026-02-22T19:49:31.903 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-22T19:49:31.908 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../092-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-22T19:49:31.909 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-22T19:49:31.923 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-pygments. 2026-02-22T19:49:31.932 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../100-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-22T19:49:31.933 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-22T19:49:31.940 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package pkg-config. 2026-02-22T19:49:31.950 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../093-pkg-config_0.29.2-1ubuntu3_amd64.deb ... 2026-02-22T19:49:31.950 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ... 2026-02-22T19:49:31.966 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-22T19:49:31.973 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-pyinotify. 2026-02-22T19:49:31.974 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../094-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-22T19:49:31.975 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-22T19:49:31.982 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../101-python3-pyinotify_0.9.6-1.3_all.deb ... 2026-02-22T19:49:31.982 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2026-02-22T19:49:31.997 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-toml. 2026-02-22T19:49:32.004 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-22T19:49:32.005 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../102-python3-toml_0.10.2-1_all.deb ... 2026-02-22T19:49:32.006 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-22T19:49:32.013 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../095-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-22T19:49:32.013 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-22T19:49:32.020 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-pytest. 2026-02-22T19:49:32.028 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-pastescript. 2026-02-22T19:49:32.029 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../103-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-22T19:49:32.029 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-22T19:49:32.036 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../096-python3-pastescript_2.0.2-4_all.deb ... 2026-02-22T19:49:32.036 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2026-02-22T19:49:32.051 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package python3-simplejson. 2026-02-22T19:49:32.054 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-pluggy. 2026-02-22T19:49:32.059 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../104-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-22T19:49:32.060 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-22T19:49:32.063 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../097-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-22T19:49:32.063 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-22T19:49:32.077 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-22T19:49:32.079 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-psutil. 2026-02-22T19:49:32.085 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../105-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-22T19:49:32.085 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-22T19:49:32.087 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../098-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-22T19:49:32.088 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-22T19:49:32.106 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-py. 2026-02-22T19:49:32.114 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../099-python3-py_1.10.0-1_all.deb ... 2026-02-22T19:49:32.115 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-22T19:49:32.135 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-pygments. 2026-02-22T19:49:32.143 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../100-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-22T19:49:32.144 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-22T19:49:32.152 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package radosgw. 2026-02-22T19:49:32.161 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../106-radosgw_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:32.162 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking radosgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:32.185 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-pyinotify. 2026-02-22T19:49:32.194 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../101-python3-pyinotify_0.9.6-1.3_all.deb ... 2026-02-22T19:49:32.194 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2026-02-22T19:49:32.208 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-toml. 2026-02-22T19:49:32.217 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../102-python3-toml_0.10.2-1_all.deb ... 2026-02-22T19:49:32.217 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-22T19:49:32.232 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-pytest. 2026-02-22T19:49:32.240 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../103-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-22T19:49:32.241 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-22T19:49:32.262 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package python3-simplejson. 2026-02-22T19:49:32.271 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../104-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-22T19:49:32.271 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-22T19:49:32.276 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package rbd-fuse. 2026-02-22T19:49:32.284 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../107-rbd-fuse_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:32.285 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking rbd-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:32.288 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-22T19:49:32.297 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../105-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-22T19:49:32.297 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-22T19:49:32.300 INFO:teuthology.orchestra.run.trial098.stdout:Selecting previously unselected package rbd-mirror. 2026-02-22T19:49:32.309 INFO:teuthology.orchestra.run.trial098.stdout:Preparing to unpack .../108-rbd-mirror_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:32.309 INFO:teuthology.orchestra.run.trial098.stdout:Unpacking rbd-mirror (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:32.365 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package radosgw. 2026-02-22T19:49:32.368 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-22T19:49:32.374 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../106-radosgw_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:32.374 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking radosgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:32.436 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-22T19:49:32.437 INFO:teuthology.orchestra.run.trial098.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-22T19:49:32.487 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package rbd-fuse. 2026-02-22T19:49:32.496 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../107-rbd-fuse_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:32.496 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking rbd-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:32.507 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T19:49:32.511 INFO:teuthology.orchestra.run.trial012.stdout:Selecting previously unselected package rbd-mirror. 2026-02-22T19:49:32.520 INFO:teuthology.orchestra.run.trial012.stdout:Preparing to unpack .../108-rbd-mirror_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T19:49:32.520 INFO:teuthology.orchestra.run.trial012.stdout:Unpacking rbd-mirror (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:32.580 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-22T19:49:32.649 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-22T19:49:32.649 INFO:teuthology.orchestra.run.trial012.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-22T19:49:32.721 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T19:49:32.780 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-22T19:49:32.985 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-22T19:49:33.148 INFO:teuthology.orchestra.run.trial098.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-22T19:49:33.159 INFO:teuthology.orchestra.run.trial098.stdout:Setting up cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:33.202 INFO:teuthology.orchestra.run.trial098.stdout:Adding system user cephadm....done 2026-02-22T19:49:33.212 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1.1) ... 2026-02-22T19:49:33.285 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-22T19:49:33.361 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-22T19:49:33.362 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-22T19:49:33.381 INFO:teuthology.orchestra.run.trial012.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-22T19:49:33.392 INFO:teuthology.orchestra.run.trial012.stdout:Setting up cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:33.433 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-22T19:49:33.435 INFO:teuthology.orchestra.run.trial012.stdout:Adding system user cephadm....done 2026-02-22T19:49:33.446 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1.1) ... 2026-02-22T19:49:33.512 INFO:teuthology.orchestra.run.trial098.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-22T19:49:33.513 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-22T19:49:33.516 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-22T19:49:33.590 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-22T19:49:33.591 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-22T19:49:33.601 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-22T19:49:33.662 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-22T19:49:33.725 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-22T19:49:33.737 INFO:teuthology.orchestra.run.trial012.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-22T19:49:33.739 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-22T19:49:33.799 INFO:teuthology.orchestra.run.trial098.stdout:Setting up unzip (6.0-26ubuntu3.2) ... 2026-02-22T19:49:33.810 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2026-02-22T19:49:33.834 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-22T19:49:33.889 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-22T19:49:33.958 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-22T19:49:33.962 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-ceph-argparse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:34.027 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-22T19:49:34.028 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-22T19:49:34.029 INFO:teuthology.orchestra.run.trial098.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2026-02-22T19:49:34.030 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ... 2026-02-22T19:49:34.031 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-22T19:49:34.032 INFO:teuthology.orchestra.run.trial098.stdout:Setting up lua5.1 (5.1.5-8.1build4) ... 2026-02-22T19:49:34.033 INFO:teuthology.orchestra.run.trial012.stdout:Setting up unzip (6.0-26ubuntu3.2) ... 2026-02-22T19:49:34.043 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2026-02-22T19:49:34.049 INFO:teuthology.orchestra.run.trial098.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode 2026-02-22T19:49:34.051 INFO:teuthology.orchestra.run.trial098.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode 2026-02-22T19:49:34.052 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-22T19:49:34.053 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-22T19:49:34.119 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-22T19:49:34.179 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-22T19:49:34.192 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-ceph-argparse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:34.258 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-22T19:49:34.261 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-22T19:49:34.262 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-22T19:49:34.263 INFO:teuthology.orchestra.run.trial012.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2026-02-22T19:49:34.264 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ... 2026-02-22T19:49:34.265 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-22T19:49:34.266 INFO:teuthology.orchestra.run.trial012.stdout:Setting up lua5.1 (5.1.5-8.1build4) ... 2026-02-22T19:49:34.283 INFO:teuthology.orchestra.run.trial012.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode 2026-02-22T19:49:34.285 INFO:teuthology.orchestra.run.trial012.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode 2026-02-22T19:49:34.286 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-22T19:49:34.287 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-22T19:49:34.335 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-22T19:49:34.410 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-22T19:49:34.424 INFO:teuthology.orchestra.run.trial098.stdout:Setting up zip (3.0-12build2) ... 2026-02-22T19:49:34.425 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-22T19:49:34.488 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-22T19:49:34.559 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-22T19:49:34.647 INFO:teuthology.orchestra.run.trial012.stdout:Setting up zip (3.0-12build2) ... 2026-02-22T19:49:34.648 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-22T19:49:34.677 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2026-02-22T19:49:34.753 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2026-02-22T19:49:34.754 INFO:teuthology.orchestra.run.trial098.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-22T19:49:34.755 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-22T19:49:34.853 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-22T19:49:34.905 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2026-02-22T19:49:34.978 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2026-02-22T19:49:34.981 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2026-02-22T19:49:34.983 INFO:teuthology.orchestra.run.trial012.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-22T19:49:34.983 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-22T19:49:35.077 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-22T19:49:35.111 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-22T19:49:35.194 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.3) ... 2026-02-22T19:49:35.205 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2026-02-22T19:49:35.302 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-22T19:49:35.339 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-22T19:49:35.377 INFO:teuthology.orchestra.run.trial098.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-22T19:49:35.378 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:35.424 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.3) ... 2026-02-22T19:49:35.464 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-22T19:49:35.532 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-22T19:49:35.607 INFO:teuthology.orchestra.run.trial012.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-22T19:49:35.608 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:35.690 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-22T19:49:35.965 INFO:teuthology.orchestra.run.trial098.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ... 2026-02-22T19:49:35.988 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:35.992 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-22T19:49:36.070 INFO:teuthology.orchestra.run.trial098.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-22T19:49:36.071 INFO:teuthology.orchestra.run.trial098.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-22T19:49:36.071 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-22T19:49:36.150 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-22T19:49:36.205 INFO:teuthology.orchestra.run.trial012.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ... 2026-02-22T19:49:36.225 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:36.226 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-22T19:49:36.227 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:36.231 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-22T19:49:36.298 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2026-02-22T19:49:36.310 INFO:teuthology.orchestra.run.trial012.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-22T19:49:36.311 INFO:teuthology.orchestra.run.trial012.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-22T19:49:36.312 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-22T19:49:36.361 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-logutils (0.3.3-8) ... 2026-02-22T19:49:36.392 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-22T19:49:36.437 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-22T19:49:36.466 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:36.467 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-22T19:49:36.515 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2026-02-22T19:49:36.547 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2026-02-22T19:49:36.591 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-22T19:49:36.621 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-logutils (0.3.3-8) ... 2026-02-22T19:49:36.674 INFO:teuthology.orchestra.run.trial098.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-22T19:49:36.675 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-22T19:49:36.698 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-22T19:49:36.762 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-22T19:49:36.763 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-22T19:49:36.776 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2026-02-22T19:49:36.844 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2026-02-22T19:49:36.850 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-22T19:49:36.923 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-22T19:49:36.929 INFO:teuthology.orchestra.run.trial012.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-22T19:49:36.930 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-22T19:49:37.009 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-22T19:49:37.016 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-22T19:49:37.017 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-22T19:49:37.084 INFO:teuthology.orchestra.run.trial098.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ... 2026-02-22T19:49:37.085 INFO:teuthology.orchestra.run.trial098.stdout:Setting up lua-sec:amd64 (1.0.2-1) ... 2026-02-22T19:49:37.086 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-22T19:49:37.087 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-22T19:49:37.094 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2026-02-22T19:49:37.173 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-22T19:49:37.228 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2026-02-22T19:49:37.258 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-22T19:49:37.308 INFO:teuthology.orchestra.run.trial098.stdout:Setting up lua-any (27ubuntu1) ... 2026-02-22T19:49:37.309 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-22T19:49:37.332 INFO:teuthology.orchestra.run.trial012.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ... 2026-02-22T19:49:37.333 INFO:teuthology.orchestra.run.trial012.stdout:Setting up lua-sec:amd64 (1.0.2-1) ... 2026-02-22T19:49:37.334 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-22T19:49:37.335 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-22T19:49:37.383 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:37.385 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-22T19:49:37.470 INFO:teuthology.orchestra.run.trial098.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-22T19:49:37.471 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-webtest (2.0.35-1) ... 2026-02-22T19:49:37.474 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2026-02-22T19:49:37.550 INFO:teuthology.orchestra.run.trial012.stdout:Setting up lua-any (27ubuntu1) ... 2026-02-22T19:49:37.551 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-22T19:49:37.555 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-22T19:49:37.622 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T19:49:37.623 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-22T19:49:37.691 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-pastescript (2.0.2-4) ... 2026-02-22T19:49:37.709 INFO:teuthology.orchestra.run.trial012.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-22T19:49:37.710 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-webtest (2.0.35-1) ... 2026-02-22T19:49:37.784 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2026-02-22T19:49:37.793 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-22T19:49:37.898 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-22T19:49:37.899 INFO:teuthology.orchestra.run.trial098.stdout:Setting up librados2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:37.900 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libsqlite3-mod-ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:37.901 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-22T19:49:37.929 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-pastescript (2.0.2-4) ... 2026-02-22T19:49:38.022 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2026-02-22T19:49:38.138 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-22T19:49:38.139 INFO:teuthology.orchestra.run.trial012.stdout:Setting up librados2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.140 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libsqlite3-mod-ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.141 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-22T19:49:38.427 INFO:teuthology.orchestra.run.trial098.stdout:Setting up luarocks (3.8.0+dfsg1-1) ... 2026-02-22T19:49:38.438 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libcephfs2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.439 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libradosstriper1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.439 INFO:teuthology.orchestra.run.trial098.stdout:Setting up librbd1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.440 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-mgr-modules-core (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.441 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.511 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-22T19:49:38.511 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-22T19:49:38.670 INFO:teuthology.orchestra.run.trial012.stdout:Setting up luarocks (3.8.0+dfsg1-1) ... 2026-02-22T19:49:38.682 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libcephfs2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.683 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libradosstriper1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.684 INFO:teuthology.orchestra.run.trial012.stdout:Setting up librbd1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.685 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-mgr-modules-core (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.686 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.758 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-22T19:49:38.758 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-22T19:49:38.890 INFO:teuthology.orchestra.run.trial098.stdout:Setting up libcephfs-dev (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.891 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-rados (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.892 INFO:teuthology.orchestra.run.trial098.stdout:Setting up librgw2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.893 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-rbd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.894 INFO:teuthology.orchestra.run.trial098.stdout:Setting up rbd-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.895 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-rgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.895 INFO:teuthology.orchestra.run.trial098.stdout:Setting up python3-cephfs (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.896 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:38.924 INFO:teuthology.orchestra.run.trial098.stdout:Adding group ceph....done 2026-02-22T19:49:38.959 INFO:teuthology.orchestra.run.trial098.stdout:Adding system user ceph....done 2026-02-22T19:49:38.968 INFO:teuthology.orchestra.run.trial098.stdout:Setting system user ceph properties....done 2026-02-22T19:49:38.975 INFO:teuthology.orchestra.run.trial098.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-22T19:49:39.062 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-22T19:49:39.132 INFO:teuthology.orchestra.run.trial012.stdout:Setting up libcephfs-dev (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.133 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-rados (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.134 INFO:teuthology.orchestra.run.trial012.stdout:Setting up librgw2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.135 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-rbd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.135 INFO:teuthology.orchestra.run.trial012.stdout:Setting up rbd-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.136 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-rgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.137 INFO:teuthology.orchestra.run.trial012.stdout:Setting up python3-cephfs (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.138 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.166 INFO:teuthology.orchestra.run.trial012.stdout:Adding group ceph....done 2026-02-22T19:49:39.201 INFO:teuthology.orchestra.run.trial012.stdout:Adding system user ceph....done 2026-02-22T19:49:39.212 INFO:teuthology.orchestra.run.trial012.stdout:Setting system user ceph properties....done 2026-02-22T19:49:39.218 INFO:teuthology.orchestra.run.trial012.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-22T19:49:39.307 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-22T19:49:39.318 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-22T19:49:39.554 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-22T19:49:39.677 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-test (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.677 INFO:teuthology.orchestra.run.trial098.stdout:Setting up radosgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.920 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-test (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.921 INFO:teuthology.orchestra.run.trial012.stdout:Setting up radosgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:39.924 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-22T19:49:39.924 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-22T19:49:40.159 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-22T19:49:40.160 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-22T19:49:40.271 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-base (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:40.391 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-22T19:49:40.528 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-base (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:40.642 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-22T19:49:40.740 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-mds (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:40.799 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-22T19:49:40.800 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-22T19:49:40.986 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-mds (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:41.044 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-22T19:49:41.044 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-22T19:49:41.158 INFO:teuthology.orchestra.run.trial098.stdout:Setting up rbd-mirror (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:41.232 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T19:49:41.232 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T19:49:41.404 INFO:teuthology.orchestra.run.trial012.stdout:Setting up rbd-mirror (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:41.480 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T19:49:41.480 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T19:49:41.589 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-mgr (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:41.664 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-22T19:49:41.664 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-22T19:49:41.856 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-mgr (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:41.935 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-22T19:49:41.936 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-22T19:49:42.012 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-osd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.097 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-22T19:49:42.097 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-22T19:49:42.286 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-osd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.370 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-22T19:49:42.371 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-22T19:49:42.468 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-mgr-k8sevents (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.469 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-mgr-diskprediction-local (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.479 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-mon (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.552 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-22T19:49:42.553 INFO:teuthology.orchestra.run.trial098.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-22T19:49:42.754 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-mgr-k8sevents (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.755 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-mgr-diskprediction-local (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.764 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-mon (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.840 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-22T19:49:42.840 INFO:teuthology.orchestra.run.trial012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-22T19:49:42.910 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-mgr-cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.922 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.923 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-mgr-dashboard (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:42.935 INFO:teuthology.orchestra.run.trial098.stdout:Setting up ceph-volume (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:43.047 INFO:teuthology.orchestra.run.trial098.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ... 2026-02-22T19:49:43.055 INFO:teuthology.orchestra.run.trial098.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-02-22T19:49:43.196 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-mgr-cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:43.208 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:43.209 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-mgr-dashboard (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:43.221 INFO:teuthology.orchestra.run.trial012.stdout:Setting up ceph-volume (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T19:49:43.298 INFO:teuthology.orchestra.run.trial098.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-22T19:49:43.331 INFO:teuthology.orchestra.run.trial012.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ... 2026-02-22T19:49:43.340 INFO:teuthology.orchestra.run.trial012.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-02-22T19:49:43.384 INFO:teuthology.orchestra.run.trial098.stdout:Processing triggers for install-info (6.8-4build1) ... 2026-02-22T19:49:43.583 INFO:teuthology.orchestra.run.trial012.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-22T19:49:43.669 INFO:teuthology.orchestra.run.trial012.stdout:Processing triggers for install-info (6.8-4build1) ... 2026-02-22T19:49:43.895 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:43.895 INFO:teuthology.orchestra.run.trial098.stdout:Running kernel seems to be up-to-date. 2026-02-22T19:49:43.895 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:43.895 INFO:teuthology.orchestra.run.trial098.stdout:The processor microcode seems to be up-to-date. 2026-02-22T19:49:43.895 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:43.895 INFO:teuthology.orchestra.run.trial098.stdout:No services need to be restarted. 2026-02-22T19:49:43.895 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:43.895 INFO:teuthology.orchestra.run.trial098.stdout:No containers need to be restarted. 2026-02-22T19:49:43.896 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:43.896 INFO:teuthology.orchestra.run.trial098.stdout:No user sessions are running outdated binaries. 2026-02-22T19:49:43.896 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:43.896 INFO:teuthology.orchestra.run.trial098.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-22T19:49:44.168 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:44.168 INFO:teuthology.orchestra.run.trial012.stdout:Running kernel seems to be up-to-date. 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout:The processor microcode seems to be up-to-date. 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout:No services need to be restarted. 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout:No containers need to be restarted. 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout:No user sessions are running outdated binaries. 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:44.169 INFO:teuthology.orchestra.run.trial012.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-22T19:49:44.710 INFO:teuthology.orchestra.run.trial098.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-22T19:49:44.717 DEBUG:teuthology.parallel:result is None 2026-02-22T19:49:44.964 INFO:teuthology.orchestra.run.trial012.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-22T19:49:44.970 DEBUG:teuthology.parallel:result is None 2026-02-22T19:49:44.971 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T19:49:45.151 DEBUG:teuthology.orchestra.run.trial012:> dpkg-query -W -f '${Version}' ceph 2026-02-22T19:49:45.161 INFO:teuthology.orchestra.run.trial012.stdout:19.2.3-689-gf90f4eae-1jammy 2026-02-22T19:49:45.162 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689-gf90f4eae-1jammy 2026-02-22T19:49:45.162 INFO:teuthology.task.install:The correct ceph version 19.2.3-689-gf90f4eae-1jammy is installed. 2026-02-22T19:49:45.165 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T19:49:45.314 DEBUG:teuthology.orchestra.run.trial098:> dpkg-query -W -f '${Version}' ceph 2026-02-22T19:49:45.324 INFO:teuthology.orchestra.run.trial098.stdout:19.2.3-689-gf90f4eae-1jammy 2026-02-22T19:49:45.324 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689-gf90f4eae-1jammy 2026-02-22T19:49:45.324 INFO:teuthology.task.install:The correct ceph version 19.2.3-689-gf90f4eae-1jammy is installed. 2026-02-22T19:49:45.327 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-22T19:49:45.327 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:45.327 DEBUG:teuthology.orchestra.run.trial012:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T19:49:45.339 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:49:45.339 DEBUG:teuthology.orchestra.run.trial098:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T19:49:45.375 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-22T19:49:45.376 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:45.376 DEBUG:teuthology.orchestra.run.trial012:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T19:49:45.389 DEBUG:teuthology.orchestra.run.trial012:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T19:49:45.440 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:49:45.440 DEBUG:teuthology.orchestra.run.trial098:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T19:49:45.449 DEBUG:teuthology.orchestra.run.trial098:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T19:49:45.498 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-22T19:49:45.499 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:45.499 DEBUG:teuthology.orchestra.run.trial012:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T19:49:45.510 DEBUG:teuthology.orchestra.run.trial012:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T19:49:45.560 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:49:45.561 DEBUG:teuthology.orchestra.run.trial098:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T19:49:45.569 DEBUG:teuthology.orchestra.run.trial098:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T19:49:45.619 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-22T19:49:45.619 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:45.619 DEBUG:teuthology.orchestra.run.trial012:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T19:49:45.629 DEBUG:teuthology.orchestra.run.trial012:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T19:49:45.680 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:49:45.681 DEBUG:teuthology.orchestra.run.trial098:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T19:49:45.689 DEBUG:teuthology.orchestra.run.trial098:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T19:49:45.738 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T19:49:45.856 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-22T19:49:45.856 DEBUG:teuthology.orchestra.run.trial012:> sudo chmod 777 /var/log/ceph 2026-02-22T19:49:45.858 DEBUG:teuthology.orchestra.run.trial098:> sudo chmod 777 /var/log/ceph 2026-02-22T19:49:45.867 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-22T19:49:45.867 DEBUG:teuthology.orchestra.run.trial012:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T19:49:45.909 DEBUG:teuthology.orchestra.run.trial098:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T19:49:45.916 INFO:tasks.ceph:Creating extra log directories... 2026-02-22T19:49:45.917 DEBUG:teuthology.orchestra.run.trial012:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T19:49:45.957 DEBUG:teuthology.orchestra.run.trial098:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T19:49:45.976 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-22T19:49:45.976 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zstd', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T19:49:45.976 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zstd supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-workunit-minimum}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64002', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zstd', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial012.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHr8AQoPmLBCDlwyLfkwSBORD6UrQX2i0NkRJcfXP05UjfcnISwAcRU/7PWmtbs6gaA2+U8uoxsT8lx4AB9jXYs=', 'trial098.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFhjjShomKSln4JXn92gq3TmrFoct33rjWGfkyU/WvmlES3UzQ3qMzMbVCSvAeKL2FJLfN0ObmSkRB/x1W++kek='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zstd', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_IMAGE_FEATURES': 'layering', 'RBD_MIRROR_CONFIG_KEY': '1', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T19:49:45.977 DEBUG:teuthology.orchestra.run.trial012:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T19:49:46.005 DEBUG:teuthology.orchestra.run.trial098:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T19:49:46.010 DEBUG:teuthology.orchestra.run.trial012:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T19:49:46.049 DEBUG:teuthology.orchestra.run.trial098:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T19:49:46.058 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:46.058 DEBUG:teuthology.orchestra.run.trial012:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T19:49:46.101 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T19:49:46.101 DEBUG:teuthology.orchestra.run.trial012:> stat /dev/vg_nvme/lv_1 2026-02-22T19:49:46.149 INFO:teuthology.orchestra.run.trial012.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T19:49:46.149 INFO:teuthology.orchestra.run.trial012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T19:49:46.149 INFO:teuthology.orchestra.run.trial012.stdout:Device: 5h/5d Inode: 906 Links: 1 2026-02-22T19:49:46.149 INFO:teuthology.orchestra.run.trial012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T19:49:46.149 INFO:teuthology.orchestra.run.trial012.stdout:Access: 2026-02-22 19:45:50.023770209 +0000 2026-02-22T19:49:46.149 INFO:teuthology.orchestra.run.trial012.stdout:Modify: 2026-02-22 19:45:49.839773337 +0000 2026-02-22T19:49:46.149 INFO:teuthology.orchestra.run.trial012.stdout:Change: 2026-02-22 19:45:49.839773337 +0000 2026-02-22T19:49:46.149 INFO:teuthology.orchestra.run.trial012.stdout: Birth: - 2026-02-22T19:49:46.149 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T19:49:46.199 INFO:teuthology.orchestra.run.trial012.stderr:1+0 records in 2026-02-22T19:49:46.199 INFO:teuthology.orchestra.run.trial012.stderr:1+0 records out 2026-02-22T19:49:46.199 INFO:teuthology.orchestra.run.trial012.stderr:512 bytes copied, 0.000178093 s, 2.9 MB/s 2026-02-22T19:49:46.200 DEBUG:teuthology.orchestra.run.trial012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T19:49:46.245 DEBUG:teuthology.orchestra.run.trial012:> stat /dev/vg_nvme/lv_2 2026-02-22T19:49:46.292 INFO:teuthology.orchestra.run.trial012.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T19:49:46.292 INFO:teuthology.orchestra.run.trial012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T19:49:46.292 INFO:teuthology.orchestra.run.trial012.stdout:Device: 5h/5d Inode: 930 Links: 1 2026-02-22T19:49:46.292 INFO:teuthology.orchestra.run.trial012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T19:49:46.292 INFO:teuthology.orchestra.run.trial012.stdout:Access: 2026-02-22 19:45:50.363764428 +0000 2026-02-22T19:49:46.292 INFO:teuthology.orchestra.run.trial012.stdout:Modify: 2026-02-22 19:45:50.191767352 +0000 2026-02-22T19:49:46.292 INFO:teuthology.orchestra.run.trial012.stdout:Change: 2026-02-22 19:45:50.191767352 +0000 2026-02-22T19:49:46.292 INFO:teuthology.orchestra.run.trial012.stdout: Birth: - 2026-02-22T19:49:46.292 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T19:49:46.339 INFO:teuthology.orchestra.run.trial012.stderr:1+0 records in 2026-02-22T19:49:46.339 INFO:teuthology.orchestra.run.trial012.stderr:1+0 records out 2026-02-22T19:49:46.339 INFO:teuthology.orchestra.run.trial012.stderr:512 bytes copied, 0.000181388 s, 2.8 MB/s 2026-02-22T19:49:46.340 DEBUG:teuthology.orchestra.run.trial012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T19:49:46.385 DEBUG:teuthology.orchestra.run.trial012:> stat /dev/vg_nvme/lv_3 2026-02-22T19:49:46.431 INFO:teuthology.orchestra.run.trial012.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T19:49:46.432 INFO:teuthology.orchestra.run.trial012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T19:49:46.432 INFO:teuthology.orchestra.run.trial012.stdout:Device: 5h/5d Inode: 952 Links: 1 2026-02-22T19:49:46.432 INFO:teuthology.orchestra.run.trial012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T19:49:46.432 INFO:teuthology.orchestra.run.trial012.stdout:Access: 2026-02-22 19:45:50.667759259 +0000 2026-02-22T19:49:46.432 INFO:teuthology.orchestra.run.trial012.stdout:Modify: 2026-02-22 19:45:50.519761775 +0000 2026-02-22T19:49:46.432 INFO:teuthology.orchestra.run.trial012.stdout:Change: 2026-02-22 19:45:50.519761775 +0000 2026-02-22T19:49:46.432 INFO:teuthology.orchestra.run.trial012.stdout: Birth: - 2026-02-22T19:49:46.432 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T19:49:46.479 INFO:teuthology.orchestra.run.trial012.stderr:1+0 records in 2026-02-22T19:49:46.479 INFO:teuthology.orchestra.run.trial012.stderr:1+0 records out 2026-02-22T19:49:46.479 INFO:teuthology.orchestra.run.trial012.stderr:512 bytes copied, 0.000177962 s, 2.9 MB/s 2026-02-22T19:49:46.480 DEBUG:teuthology.orchestra.run.trial012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T19:49:46.525 DEBUG:teuthology.orchestra.run.trial012:> stat /dev/vg_nvme/lv_4 2026-02-22T19:49:46.571 INFO:teuthology.orchestra.run.trial012.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T19:49:46.572 INFO:teuthology.orchestra.run.trial012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T19:49:46.572 INFO:teuthology.orchestra.run.trial012.stdout:Device: 5h/5d Inode: 979 Links: 1 2026-02-22T19:49:46.572 INFO:teuthology.orchestra.run.trial012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T19:49:46.572 INFO:teuthology.orchestra.run.trial012.stdout:Access: 2026-02-22 19:45:50.835756404 +0000 2026-02-22T19:49:46.572 INFO:teuthology.orchestra.run.trial012.stdout:Modify: 2026-02-22 19:45:50.831756471 +0000 2026-02-22T19:49:46.572 INFO:teuthology.orchestra.run.trial012.stdout:Change: 2026-02-22 19:45:50.831756471 +0000 2026-02-22T19:49:46.572 INFO:teuthology.orchestra.run.trial012.stdout: Birth: - 2026-02-22T19:49:46.572 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T19:49:46.620 INFO:teuthology.orchestra.run.trial012.stderr:1+0 records in 2026-02-22T19:49:46.620 INFO:teuthology.orchestra.run.trial012.stderr:1+0 records out 2026-02-22T19:49:46.620 INFO:teuthology.orchestra.run.trial012.stderr:512 bytes copied, 0.000161382 s, 3.2 MB/s 2026-02-22T19:49:46.621 DEBUG:teuthology.orchestra.run.trial012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T19:49:46.669 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T19:49:46.669 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial012.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T19:49:46.669 INFO:tasks.ceph:Generating config... 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:46.670 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:46.671 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:46.671 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:46.671 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:46.671 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:46.671 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T19:49:46.671 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T19:49:46.671 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T19:49:46.671 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T19:49:46.671 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T19:49:46.671 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T19:49:46.671 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T19:49:46.671 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] bluestore compression algorithm = zstd 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T19:49:46.671 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T19:49:46.671 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-22T19:49:46.671 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-22T19:49:46.728 INFO:teuthology.orchestra.run.trial012.stdout:creating /etc/ceph/cluster1.keyring 2026-02-22T19:49:46.731 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-22T19:49:46.791 DEBUG:teuthology.orchestra.run.trial012:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T19:49:46.839 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.12')] 2026-02-22T19:49:46.839 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.12', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zstd', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T19:49:46.840 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:46.840 DEBUG:teuthology.orchestra.run.trial012:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T19:49:46.884 DEBUG:teuthology.orchestra.run.trial012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.12 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T19:49:46.940 INFO:teuthology.orchestra.run.trial012.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T19:49:46.940 INFO:teuthology.orchestra.run.trial012.stdout:monmaptool: generated fsid 1fbaf3f1-4414-4c85-a76d-328a2a8144bc 2026-02-22T19:49:46.940 INFO:teuthology.orchestra.run.trial012.stdout:setting min_mon_release = quincy 2026-02-22T19:49:46.940 INFO:teuthology.orchestra.run.trial012.stdout:epoch 0 2026-02-22T19:49:46.940 INFO:teuthology.orchestra.run.trial012.stdout:fsid 1fbaf3f1-4414-4c85-a76d-328a2a8144bc 2026-02-22T19:49:46.940 INFO:teuthology.orchestra.run.trial012.stdout:last_changed 2026-02-22T19:49:46.940073+0000 2026-02-22T19:49:46.941 INFO:teuthology.orchestra.run.trial012.stdout:created 2026-02-22T19:49:46.940073+0000 2026-02-22T19:49:46.941 INFO:teuthology.orchestra.run.trial012.stdout:min_mon_release 17 (quincy) 2026-02-22T19:49:46.941 INFO:teuthology.orchestra.run.trial012.stdout:election_strategy: 1 2026-02-22T19:49:46.941 INFO:teuthology.orchestra.run.trial012.stdout:0: [v2:10.20.193.12:3300/0,v1:10.20.193.12:6789/0] mon.a 2026-02-22T19:49:46.941 INFO:teuthology.orchestra.run.trial012.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-22T19:49:46.942 DEBUG:teuthology.orchestra.run.trial012:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T19:49:46.987 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 1fbaf3f1-4414-4c85-a76d-328a2a8144bc... 2026-02-22T19:49:46.988 DEBUG:teuthology.orchestra.run.trial012:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T19:49:47.029 DEBUG:teuthology.orchestra.run.trial098:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout:[global] 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: chdir = "" 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: auth supported = cephx 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: filestore xattr use omap = true 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: mon clock drift allowed = 1.000 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: osd crush chooseleaf type = 0 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: auth debug = true 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: ms die on old message = true 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: ms die on bug = true 2026-02-22T19:49:47.049 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon pg warn max object skew = 0 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: osd pool default size = 2 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon osd allow primary affinity = true 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon osd allow pg remap = true 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on legacy crush tunables = false 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on crush straw calc version zero = false 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on no sortbitwise = false 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on osd down out interval zero = false 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on too few osds = false 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon_allow_pool_size_one = true 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: osd default data pool replay window = 5 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon allow pool delete = true 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon cluster log file level = debug 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: debug asserts on shutdown = true 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon health detail to clog = false 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon host = 10.20.193.12 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon client directed command retry = 5 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on pool no app = False 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: ms inject socket failures = 5000 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: fsid = 1fbaf3f1-4414-4c85-a76d-328a2a8144bc 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout:[osd] 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: osd journal size = 100 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: osd scrub load threshold = 5.0 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: osd scrub max interval = 600 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: osd mclock profile = high_recovery_ops 2026-02-22T19:49:47.050 INFO:teuthology.orchestra.run.trial012.stdout: osd mclock skip benchmark = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd recover clone overlap = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd recovery max chunk = 1048576 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd debug shutdown = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd debug op order = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd debug verify stray on activate = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd debug trim objects = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd open classes on start = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd debug pg log writeout = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd deep scrub update digest min age = 30 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd map max advance = 10 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: journal zero on create = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: filestore ondisk finisher threads = 3 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: filestore apply finisher threads = 3 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: bdev debug aio = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd debug misdirected ops = true 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: bluestore block size = 96636764160 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: bluestore compression algorithm = zstd 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: bluestore compression mode = aggressive 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: bluestore fsck on mount = True 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: debug bluefs = 1/20 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: debug bluestore = 1/20 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: debug ms = 1 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: debug osd = 20 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: debug rocksdb = 4/10 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: mon osd full ratio = 0.9 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: mon osd nearfull ratio = 0.8 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd failsafe full ratio = 0.95 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: osd objectstore = bluestore 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout:[mgr] 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: debug ms = 1 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: debug mgr = 20 2026-02-22T19:49:47.051 INFO:teuthology.orchestra.run.trial012.stdout: debug mon = 20 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: debug auth = 20 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon reweight min pgs per osd = 4 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon reweight min bytes per osd = 10 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mgr/telemetry/nag = false 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout:[mon] 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: debug ms = 1 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: debug mon = 20 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: debug paxos = 20 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: debug auth = 20 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon data avail warn = 5 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon mgr mkfs grace = 240 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon reweight min pgs per osd = 4 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon osd reporter subtree level = osd 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon osd prime pg temp = true 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon reweight min bytes per osd = 10 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: # 1m isn't quite enough 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon_down_mkfs_grace = 2m 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_filestore_osds = false 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout:[client] 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: rgw cache enabled = true 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: rgw enable ops log = true 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: rgw enable usage log = true 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: debug rbd = 20 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: debug rbd_mirror = 15 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: log to stderr = False 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: rbd default features = 125 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout:[mon.a] 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.0] 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.052 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.1] 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.2] 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.3] 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.4] 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.5] 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.6] 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout:[global] 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: chdir = "" 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: auth supported = cephx 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: filestore xattr use omap = true 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: mon clock drift allowed = 1.000 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: osd crush chooseleaf type = 0 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: auth debug = true 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: ms die on old message = true 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: ms die on bug = true 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: mon pg warn max object skew = 0 2026-02-22T19:49:47.053 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd pool default size = 2 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon osd allow primary affinity = true 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon osd allow pg remap = true 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on legacy crush tunables = false 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on crush straw calc version zero = false 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on no sortbitwise = false 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on osd down out interval zero = false 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on too few osds = false 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon_allow_pool_size_one = true 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd default data pool replay window = 5 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon allow pool delete = true 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon cluster log file level = debug 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: debug asserts on shutdown = true 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon health detail to clog = false 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon host = 10.20.193.12 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon client directed command retry = 5 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on pool no app = False 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: ms inject socket failures = 5000 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: fsid = 1fbaf3f1-4414-4c85-a76d-328a2a8144bc 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout:[osd] 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd journal size = 100 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd scrub load threshold = 5.0 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd scrub max interval = 600 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd mclock profile = high_recovery_ops 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd mclock skip benchmark = true 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd recover clone overlap = true 2026-02-22T19:49:47.054 INFO:teuthology.orchestra.run.trial098.stdout: osd recovery max chunk = 1048576 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd debug shutdown = true 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd debug op order = true 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd debug verify stray on activate = true 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd debug trim objects = true 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd open classes on start = true 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd debug pg log writeout = true 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd deep scrub update digest min age = 30 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd map max advance = 10 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: journal zero on create = true 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: filestore ondisk finisher threads = 3 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: filestore apply finisher threads = 3 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: bdev debug aio = true 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd debug misdirected ops = true 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: bluestore block size = 96636764160 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: bluestore compression algorithm = zstd 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: bluestore compression mode = aggressive 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: bluestore fsck on mount = True 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug bluefs = 1/20 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug bluestore = 1/20 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug ms = 1 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug osd = 20 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug rocksdb = 4/10 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: mon osd full ratio = 0.9 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: mon osd nearfull ratio = 0.8 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd failsafe full ratio = 0.95 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: osd objectstore = bluestore 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout:[mgr] 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug ms = 1 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug mgr = 20 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug mon = 20 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug auth = 20 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: mon reweight min pgs per osd = 4 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: mon reweight min bytes per osd = 10 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: mgr/telemetry/nag = false 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout:[mon] 2026-02-22T19:49:47.055 INFO:teuthology.orchestra.run.trial098.stdout: debug ms = 1 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: debug mon = 20 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: debug paxos = 20 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: debug auth = 20 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon data avail warn = 5 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon mgr mkfs grace = 240 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon reweight min pgs per osd = 4 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon osd reporter subtree level = osd 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon osd prime pg temp = true 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon reweight min bytes per osd = 10 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: # 1m isn't quite enough 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon_down_mkfs_grace = 2m 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_filestore_osds = false 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout:[client] 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: rgw cache enabled = true 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: rgw enable ops log = true 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: rgw enable usage log = true 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: debug rbd = 20 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: debug rbd_mirror = 15 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: log to stderr = False 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: rbd default features = 125 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout:[mon.a] 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.0] 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.1] 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.2] 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.056 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.3] 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.4] 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.5] 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.6] 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:49:47.057 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:49:47.059 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-22T19:49:47.059 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-22T19:49:47.126 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T19:49:47.126 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:47.126 DEBUG:teuthology.orchestra.run.trial012:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-22T19:49:47.175 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:47.175 DEBUG:teuthology.orchestra.run.trial012:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-22T19:49:47.219 INFO:tasks.ceph:Sending monmap to node ubuntu@trial012.front.sepia.ceph.com 2026-02-22T19:49:47.220 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:47.220 DEBUG:teuthology.orchestra.run.trial012:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T19:49:47.220 DEBUG:teuthology.orchestra.run.trial012:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T19:49:47.273 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:47.273 DEBUG:teuthology.orchestra.run.trial012:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T19:49:47.320 INFO:tasks.ceph:Sending monmap to node ubuntu@trial098.front.sepia.ceph.com 2026-02-22T19:49:47.320 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:49:47.320 DEBUG:teuthology.orchestra.run.trial098:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T19:49:47.320 DEBUG:teuthology.orchestra.run.trial098:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T19:49:47.334 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:49:47.334 DEBUG:teuthology.orchestra.run.trial098:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T19:49:47.378 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T19:49:47.378 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T19:49:47.378 DEBUG:teuthology.orchestra.run.trial012:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T19:49:47.404 INFO:teuthology.orchestra.run.trial012.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T19:49:47.407 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T19:49:47.407 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T19:49:47.407 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-22T19:49:47.471 INFO:teuthology.orchestra.run.trial012.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-22T19:49:47.478 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T19:49:47.497 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T19:49:47.505 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T19:49:47.564 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T19:49:47.571 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T19:49:47.633 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T19:49:47.640 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T19:49:47.704 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T19:49:47.712 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T19:49:47.776 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T19:49:47.784 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T19:49:47.842 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T19:49:47.849 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T19:49:47.907 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T19:49:47.913 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T19:49:47.971 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T19:49:47.978 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T19:49:47.978 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial012.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T19:49:47.978 DEBUG:teuthology.orchestra.run.trial012:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-22T19:49:47.986 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T19:49:47.986 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-22T19:49:47.986 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial012.front.sepia.ceph.com 2026-02-22T19:49:47.986 DEBUG:teuthology.orchestra.run.trial012:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T19:49:48.037 INFO:teuthology.orchestra.run.trial012.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T19:49:48.038 INFO:teuthology.orchestra.run.trial012.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T19:49:48.038 INFO:teuthology.orchestra.run.trial012.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T19:49:48.038 INFO:teuthology.orchestra.run.trial012.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T19:49:48.038 INFO:teuthology.orchestra.run.trial012.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T19:49:48.038 INFO:teuthology.orchestra.run.trial012.stdout: = sunit=0 swidth=0 blks 2026-02-22T19:49:48.038 INFO:teuthology.orchestra.run.trial012.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T19:49:48.038 INFO:teuthology.orchestra.run.trial012.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T19:49:48.038 INFO:teuthology.orchestra.run.trial012.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T19:49:48.038 INFO:teuthology.orchestra.run.trial012.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T19:49:48.626 INFO:teuthology.orchestra.run.trial012.stdout:Discarding blocks...Done. 2026-02-22T19:49:48.628 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial012.front.sepia.ceph.com -o noatime 2026-02-22T19:49:48.628 DEBUG:teuthology.orchestra.run.trial012:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-22T19:49:48.684 DEBUG:teuthology.orchestra.run.trial012:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-22T19:49:48.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T19:49:48.735 INFO:teuthology.orchestra.run.trial012.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T19:49:48.736 DEBUG:teuthology.orchestra.run.trial012:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-22T19:49:48.785 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T19:49:48.785 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-22T19:49:48.785 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial012.front.sepia.ceph.com 2026-02-22T19:49:48.785 DEBUG:teuthology.orchestra.run.trial012:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T19:49:48.838 INFO:teuthology.orchestra.run.trial012.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T19:49:48.839 INFO:teuthology.orchestra.run.trial012.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T19:49:48.839 INFO:teuthology.orchestra.run.trial012.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T19:49:48.839 INFO:teuthology.orchestra.run.trial012.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T19:49:48.839 INFO:teuthology.orchestra.run.trial012.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T19:49:48.839 INFO:teuthology.orchestra.run.trial012.stdout: = sunit=0 swidth=0 blks 2026-02-22T19:49:48.839 INFO:teuthology.orchestra.run.trial012.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T19:49:48.839 INFO:teuthology.orchestra.run.trial012.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T19:49:48.839 INFO:teuthology.orchestra.run.trial012.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T19:49:48.839 INFO:teuthology.orchestra.run.trial012.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T19:49:49.402 INFO:teuthology.orchestra.run.trial012.stdout:Discarding blocks...Done. 2026-02-22T19:49:49.404 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial012.front.sepia.ceph.com -o noatime 2026-02-22T19:49:49.404 DEBUG:teuthology.orchestra.run.trial012:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-22T19:49:49.462 DEBUG:teuthology.orchestra.run.trial012:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-22T19:49:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T19:49:49.511 INFO:teuthology.orchestra.run.trial012.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T19:49:49.512 DEBUG:teuthology.orchestra.run.trial012:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-22T19:49:49.561 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T19:49:49.561 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-22T19:49:49.561 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial012.front.sepia.ceph.com 2026-02-22T19:49:49.561 DEBUG:teuthology.orchestra.run.trial012:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T19:49:49.614 INFO:teuthology.orchestra.run.trial012.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T19:49:49.614 INFO:teuthology.orchestra.run.trial012.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T19:49:49.615 INFO:teuthology.orchestra.run.trial012.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T19:49:49.615 INFO:teuthology.orchestra.run.trial012.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T19:49:49.615 INFO:teuthology.orchestra.run.trial012.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T19:49:49.615 INFO:teuthology.orchestra.run.trial012.stdout: = sunit=0 swidth=0 blks 2026-02-22T19:49:49.615 INFO:teuthology.orchestra.run.trial012.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T19:49:49.615 INFO:teuthology.orchestra.run.trial012.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T19:49:49.615 INFO:teuthology.orchestra.run.trial012.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T19:49:49.615 INFO:teuthology.orchestra.run.trial012.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T19:49:50.187 INFO:teuthology.orchestra.run.trial012.stdout:Discarding blocks...Done. 2026-02-22T19:49:50.188 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial012.front.sepia.ceph.com -o noatime 2026-02-22T19:49:50.189 DEBUG:teuthology.orchestra.run.trial012:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-22T19:49:50.245 DEBUG:teuthology.orchestra.run.trial012:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-22T19:49:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T19:49:50.295 INFO:teuthology.orchestra.run.trial012.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T19:49:50.295 DEBUG:teuthology.orchestra.run.trial012:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T19:49:50.358 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:50.353+0000 7f796f3a88c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-22T19:49:50.359 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:50.353+0000 7f796f3a88c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-22T19:49:50.359 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:50.353+0000 7f796f3a88c0 -1 bdev(0x55d3b0969800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-22T19:49:50.359 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:50.353+0000 7f796f3a88c0 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-22T19:49:53.444 DEBUG:teuthology.orchestra.run.trial012:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-22T19:49:53.493 DEBUG:teuthology.orchestra.run.trial012:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T19:49:53.557 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:53.553+0000 7fe41af6a8c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-22T19:49:53.558 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:53.553+0000 7fe41af6a8c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-22T19:49:53.558 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:53.553+0000 7fe41af6a8c0 -1 bdev(0x559fb4721800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-22T19:49:53.558 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:53.553+0000 7fe41af6a8c0 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-22T19:49:56.660 DEBUG:teuthology.orchestra.run.trial012:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-22T19:49:56.709 DEBUG:teuthology.orchestra.run.trial012:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T19:49:56.773 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:56.769+0000 7fca7659e8c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-22T19:49:56.774 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:56.769+0000 7fca7659e8c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-22T19:49:56.774 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:56.769+0000 7fca7659e8c0 -1 bdev(0x562765a67800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-22T19:49:56.774 INFO:teuthology.orchestra.run.trial012.stderr:2026-02-22T19:49:56.769+0000 7fca7659e8c0 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-22T19:49:59.916 DEBUG:teuthology.orchestra.run.trial012:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-22T19:49:59.965 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T19:49:59.966 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:49:59.966 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-22T19:50:00.016 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:00.017 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-22T19:50:00.063 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:00.063 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-22T19:50:00.112 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:00.112 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-22T19:50:00.160 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:00.161 DEBUG:teuthology.orchestra.run.trial012:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-22T19:50:00.205 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:00.205 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-22T19:50:00.211 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:00.211 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-22T19:50:00.255 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:00.255 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-22T19:50:00.299 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:00.299 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-22T19:50:00.347 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:00.347 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-22T19:50:00.395 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:00.395 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-22T19:50:00.439 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:00.439 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-22T19:50:00.483 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:00.483 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-22T19:50:00.527 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T19:50:00.527 DEBUG:teuthology.orchestra.run.trial012:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout:[mgr.x] 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtprxgZGBAAR22fXOTz/MgWb2BImefTXA== 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout:[osd.0] 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDeXZtpj0RMFRAA7LShtw/fH5uggN+BaF9ddA== 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout:[osd.1] 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDhXZtpmkEvIRAA2Nk4FRduhIwjwPYjCyV0bg== 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout:[osd.2] 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDkXZtpnacVLhAAeuKIRUVnsBZHD9d2xP/x2w== 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout:[client.0] 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtpfOcJHBAA62SProqCIHW1hYgLpIa6vw== 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror] 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtpcC64HRAA540s/XyzUc0NJrT55rLP+Q== 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.0] 2026-02-22T19:50:00.536 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtpFse2IRAA/8HhEZINBaoNlABmgrGWcg== 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.1] 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtpDj7WJRAA7dG2JSHoKBqmdIKDgthDMg== 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.2] 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtpQygaKhAAKjzPZCa0M8bH6Y40zAJz0g== 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.3] 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtp2IpmLhAAmkI1Mkj+iXmUJVesqc7JKg== 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.4] 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtpUBlWMhAA3+lf2FUjsi1BBVP5wbadTg== 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.5] 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtp5XMoNhAAj9W31gTaF5YARYUznYCj+w== 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.6] 2026-02-22T19:50:00.537 INFO:teuthology.orchestra.run.trial012.stdout: key = AQDbXZtpCm/5ORAAPbmb6hO+u30HxakLvq69Tw== 2026-02-22T19:50:00.538 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T19:50:00.602 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T19:50:00.665 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T19:50:00.729 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T19:50:00.793 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:00.857 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:00.921 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:00.985 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:01.050 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:01.119 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:01.183 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:01.251 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:01.314 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:01.382 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T19:50:01.383 DEBUG:teuthology.orchestra.run.trial012:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-22T19:50:01.432 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-22T19:50:01.507 DEBUG:teuthology.orchestra.run.trial012:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-22T19:50:01.560 DEBUG:teuthology.orchestra.run.trial012:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T19:50:01.612 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-22T19:50:01.612 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-22T19:50:01.612 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-22T19:50:01.654 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-22T19:50:01.654 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-22T19:50:01.654 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-22T19:50:01.655 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-22T19:50:01.657 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-22T19:50:01.657 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T19:50:01.657 DEBUG:teuthology.orchestra.run.trial012:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-22T19:50:01.780 INFO:teuthology.orchestra.run.trial012.stderr:adjusted tunables profile to default 2026-02-22T19:50:01.790 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T19:50:01.791 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-22T19:50:01.791 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:01.791 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-22T19:50:01.801 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:01.801 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-22T19:50:01.854 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:01.854 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-22T19:50:01.904 DEBUG:teuthology.orchestra.run.trial012:> sudo ceph --cluster cluster1 osd new 918567e3-88de-4556-bdb6-f74f987b785a 0 2026-02-22T19:50:01.982 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:01.977+0000 7fddbd0e2280 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T19:50:02.009 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.005+0000 7fddbd0e2280 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T19:50:02.041 INFO:teuthology.orchestra.run.trial012.stdout:0 2026-02-22T19:50:02.047 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.041+0000 7fddbd0e2280 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T19:50:02.051 DEBUG:teuthology.orchestra.run.trial012:> sudo ceph --cluster cluster1 osd new 13b0e3fb-8b1f-4417-b735-cd1accb01bc2 1 2026-02-22T19:50:02.130 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.125+0000 7fddbd0e2280 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T19:50:02.187 INFO:teuthology.orchestra.run.trial012.stdout:1 2026-02-22T19:50:02.197 DEBUG:teuthology.orchestra.run.trial012:> sudo ceph --cluster cluster1 osd new 079aba72-a855-44a4-9127-c34c8eafdc06 2 2026-02-22T19:50:02.212 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.205+0000 7fddbd0e2280 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T19:50:02.251 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.245+0000 7fddbd0e2280 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T19:50:02.309 INFO:teuthology.orchestra.run.trial012.stdout:2 2026-02-22T19:50:02.318 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-22T19:50:02.319 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-22T19:50:02.328 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.321+0000 7fddbd0e2280 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T19:50:02.330 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-22T19:50:02.330 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-22T19:50:02.330 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-22T19:50:02.332 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-22T19:50:02.332 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-22T19:50:02.332 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-22T19:50:02.334 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-22T19:50:02.335 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T19:50:02.433 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:02.434 INFO:teuthology.orchestra.run.trial012.stdout:{"epoch":5,"fsid":"1fbaf3f1-4414-4c85-a76d-328a2a8144bc","created":"2026-02-22T19:50:01.737667+0000","modified":"2026-02-22T19:50:02.307709+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T19:50:02.307709+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"918567e3-88de-4556-bdb6-f74f987b785a","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"13b0e3fb-8b1f-4417-b735-cd1accb01bc2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"079aba72-a855-44a4-9127-c34c8eafdc06","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:02.443 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-22T19:50:02.444 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T19:50:02.512 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.505+0000 7fddbd0e2280 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T19:50:02.746 DEBUG:teuthology.orchestra.run.trial012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T19:50:02.820 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.813+0000 7fddbd0e2280 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T19:50:02.835 INFO:teuthology.misc.health.trial012.stdout: 2026-02-22T19:50:02.835 INFO:teuthology.misc.health.trial012.stdout:{"epoch":5,"fsid":"1fbaf3f1-4414-4c85-a76d-328a2a8144bc","created":"2026-02-22T19:50:01.737667+0000","modified":"2026-02-22T19:50:02.307709+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T19:50:02.307709+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"918567e3-88de-4556-bdb6-f74f987b785a","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"13b0e3fb-8b1f-4417-b735-cd1accb01bc2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"079aba72-a855-44a4-9127-c34c8eafdc06","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:02.843 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T19:50:02.914 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.909+0000 7fddbd0e2280 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T19:50:02.942 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.937+0000 7fddbd0e2280 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T19:50:02.973 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:02.969+0000 7fddbd0e2280 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T19:50:03.015 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.009+0000 7fddbd0e2280 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T19:50:03.116 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.109+0000 7fddbd0e2280 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T19:50:03.307 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-22T19:50:03.478 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.473+0000 7fddbd0e2280 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T19:50:03.509 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.505+0000 7fddbd0e2280 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T19:50:03.601 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.597+0000 7fddbd0e2280 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T19:50:03.625 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.621+0000 7fddbd0e2280 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T19:50:03.652 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.645+0000 7fddbd0e2280 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T19:50:03.679 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.673+0000 7fddbd0e2280 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T19:50:03.797 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T19:50:03.798 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T19:50:03.798 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr: from numpy import show_config as show_numpy_config 2026-02-22T19:50:03.804 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.797+0000 7fddbd0e2280 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T19:50:03.830 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.825+0000 7fddbd0e2280 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T19:50:03.868 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:03.861+0000 7fddbd0e2280 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T19:50:03.925 INFO:tasks.ceph.cluster1.osd.1.trial012.stderr:2026-02-22T19:50:03.921+0000 7f694a4238c0 -1 Falling back to public interface 2026-02-22T19:50:04.410 INFO:tasks.ceph.cluster1.osd.2.trial012.stderr:2026-02-22T19:50:04.405+0000 7f4662c438c0 -1 Falling back to public interface 2026-02-22T19:50:04.433 INFO:tasks.ceph.cluster1.osd.0.trial012.stderr:2026-02-22T19:50:04.429+0000 7f6d0fffe8c0 -1 Falling back to public interface 2026-02-22T19:50:04.892 INFO:tasks.ceph.cluster1.mgr.x.trial012.stderr:2026-02-22T19:50:04.885+0000 7fdcdb931640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T19:50:07.768 INFO:tasks.ceph.cluster1.osd.1.trial012.stderr:2026-02-22T19:50:07.761+0000 7f694a4238c0 -1 osd.1 0 log_to_monitors true 2026-02-22T19:50:08.490 INFO:tasks.ceph.cluster1.osd.2.trial012.stderr:2026-02-22T19:50:08.485+0000 7f4662c438c0 -1 osd.2 0 log_to_monitors true 2026-02-22T19:50:08.531 INFO:tasks.ceph.cluster1.osd.0.trial012.stderr:2026-02-22T19:50:08.525+0000 7f6d0fffe8c0 -1 osd.0 0 log_to_monitors true 2026-02-22T19:50:08.900 INFO:tasks.ceph.cluster1.osd.1.trial012.stderr:2026-02-22T19:50:08.893+0000 7f69463cc640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T19:50:08.925 INFO:tasks.ceph.cluster1.osd.1.trial012.stderr:2026-02-22T19:50:08.921+0000 7f693e1ff640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T19:50:09.146 DEBUG:teuthology.orchestra.run.trial012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T19:50:09.290 INFO:teuthology.misc.health.trial012.stdout: 2026-02-22T19:50:09.291 INFO:teuthology.misc.health.trial012.stdout:{"epoch":7,"fsid":"1fbaf3f1-4414-4c85-a76d-328a2a8144bc","created":"2026-02-22T19:50:01.737667+0000","modified":"2026-02-22T19:50:08.892443+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T19:50:02.307709+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"918567e3-88de-4556-bdb6-f74f987b785a","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"13b0e3fb-8b1f-4417-b735-cd1accb01bc2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"079aba72-a855-44a4-9127-c34c8eafdc06","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:09.300 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T19:50:09.902 INFO:tasks.ceph.cluster1.osd.0.trial012.stderr:2026-02-22T19:50:09.897+0000 7f6d0bfa7640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T19:50:09.904 INFO:tasks.ceph.cluster1.osd.2.trial012.stderr:2026-02-22T19:50:09.897+0000 7f465ebec640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T19:50:09.930 INFO:tasks.ceph.cluster1.osd.0.trial012.stderr:2026-02-22T19:50:09.925+0000 7f6d03dda640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T19:50:09.930 INFO:tasks.ceph.cluster1.osd.2.trial012.stderr:2026-02-22T19:50:09.925+0000 7f4656a1f640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T19:50:15.603 DEBUG:teuthology.orchestra.run.trial012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T19:50:15.745 INFO:teuthology.misc.health.trial012.stdout: 2026-02-22T19:50:15.745 INFO:teuthology.misc.health.trial012.stdout:{"epoch":13,"fsid":"1fbaf3f1-4414-4c85-a76d-328a2a8144bc","created":"2026-02-22T19:50:01.737667+0000","modified":"2026-02-22T19:50:14.908170+0000","last_up_change":"2026-02-22T19:50:10.899363+0000","last_in_change":"2026-02-22T19:50:02.307709+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T19:50:11.921200+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"918567e3-88de-4556-bdb6-f74f987b785a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6818","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6819","nonce":4046618351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6820","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6821","nonce":4046618351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6824","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6825","nonce":4046618351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6822","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6823","nonce":4046618351}]},"public_addr":"10.20.193.12:6819/4046618351","cluster_addr":"10.20.193.12:6821/4046618351","heartbeat_back_addr":"10.20.193.12:6825/4046618351","heartbeat_front_addr":"10.20.193.12:6823/4046618351","state":["exists","up"]},{"osd":1,"uuid":"13b0e3fb-8b1f-4417-b735-cd1accb01bc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6802","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6803","nonce":1224390038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6804","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6805","nonce":1224390038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6808","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6809","nonce":1224390038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6806","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6807","nonce":1224390038}]},"public_addr":"10.20.193.12:6803/1224390038","cluster_addr":"10.20.193.12:6805/1224390038","heartbeat_back_addr":"10.20.193.12:6809/1224390038","heartbeat_front_addr":"10.20.193.12:6807/1224390038","state":["exists","up"]},{"osd":2,"uuid":"079aba72-a855-44a4-9127-c34c8eafdc06","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6810","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6811","nonce":1321751267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6812","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6813","nonce":1321751267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6816","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6817","nonce":1321751267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6814","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6815","nonce":1321751267}]},"public_addr":"10.20.193.12:6811/1321751267","cluster_addr":"10.20.193.12:6813/1321751267","heartbeat_back_addr":"10.20.193.12:6817/1321751267","heartbeat_front_addr":"10.20.193.12:6815/1321751267","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:09.575529+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:08.778690+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:09.480453+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:15.754 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T19:50:15.754 INFO:tasks.ceph:Creating RBD pool 2026-02-22T19:50:15.755 DEBUG:teuthology.orchestra.run.trial012:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-22T19:50:15.913 INFO:teuthology.orchestra.run.trial012.stderr:pool 'rbd' created 2026-02-22T19:50:15.924 DEBUG:teuthology.orchestra.run.trial012:> rbd --cluster cluster1 pool init rbd 2026-02-22T19:50:18.929 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-22T19:50:18.929 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-22T19:50:18.930 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-22T19:50:18.930 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-22T19:50:18.930 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T19:50:19.096 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:19.105 INFO:teuthology.orchestra.run.trial012.stdout:{"epoch":5,"flags":0,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6800","nonce":798536585},{"type":"v1","addr":"10.20.193.12:6801","nonce":798536585}]},"active_addr":"10.20.193.12:6801/798536585","active_change":"2026-02-22T19:50:03.884939+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.12:0","nonce":1364782626}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.12:0","nonce":2771747971}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.12:0","nonce":1605500734}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.12:0","nonce":2055658913}]}]} 2026-02-22T19:50:19.109 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-22T19:50:19.109 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-22T19:50:19.110 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T19:50:19.258 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:19.258 INFO:teuthology.orchestra.run.trial012.stdout:{"epoch":17,"fsid":"1fbaf3f1-4414-4c85-a76d-328a2a8144bc","created":"2026-02-22T19:50:01.737667+0000","modified":"2026-02-22T19:50:18.917545+0000","last_up_change":"2026-02-22T19:50:10.899363+0000","last_in_change":"2026-02-22T19:50:02.307709+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T19:50:11.921200+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T19:50:15.900092+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"918567e3-88de-4556-bdb6-f74f987b785a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6818","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6819","nonce":4046618351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6820","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6821","nonce":4046618351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6824","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6825","nonce":4046618351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6822","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6823","nonce":4046618351}]},"public_addr":"10.20.193.12:6819/4046618351","cluster_addr":"10.20.193.12:6821/4046618351","heartbeat_back_addr":"10.20.193.12:6825/4046618351","heartbeat_front_addr":"10.20.193.12:6823/4046618351","state":["exists","up"]},{"osd":1,"uuid":"13b0e3fb-8b1f-4417-b735-cd1accb01bc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6802","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6803","nonce":1224390038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6804","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6805","nonce":1224390038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6808","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6809","nonce":1224390038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6806","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6807","nonce":1224390038}]},"public_addr":"10.20.193.12:6803/1224390038","cluster_addr":"10.20.193.12:6805/1224390038","heartbeat_back_addr":"10.20.193.12:6809/1224390038","heartbeat_front_addr":"10.20.193.12:6807/1224390038","state":["exists","up"]},{"osd":2,"uuid":"079aba72-a855-44a4-9127-c34c8eafdc06","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6810","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6811","nonce":1321751267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6812","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6813","nonce":1321751267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6816","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6817","nonce":1321751267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6814","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6815","nonce":1321751267}]},"public_addr":"10.20.193.12:6811/1321751267","cluster_addr":"10.20.193.12:6813/1321751267","heartbeat_back_addr":"10.20.193.12:6817/1321751267","heartbeat_front_addr":"10.20.193.12:6815/1321751267","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:09.575529+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:08.778690+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:09.480453+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:19.268 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-22T19:50:19.268 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T19:50:19.420 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:19.421 INFO:teuthology.orchestra.run.trial012.stdout:{"epoch":17,"fsid":"1fbaf3f1-4414-4c85-a76d-328a2a8144bc","created":"2026-02-22T19:50:01.737667+0000","modified":"2026-02-22T19:50:18.917545+0000","last_up_change":"2026-02-22T19:50:10.899363+0000","last_in_change":"2026-02-22T19:50:02.307709+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T19:50:11.921200+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T19:50:15.900092+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"918567e3-88de-4556-bdb6-f74f987b785a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6818","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6819","nonce":4046618351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6820","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6821","nonce":4046618351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6824","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6825","nonce":4046618351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6822","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6823","nonce":4046618351}]},"public_addr":"10.20.193.12:6819/4046618351","cluster_addr":"10.20.193.12:6821/4046618351","heartbeat_back_addr":"10.20.193.12:6825/4046618351","heartbeat_front_addr":"10.20.193.12:6823/4046618351","state":["exists","up"]},{"osd":1,"uuid":"13b0e3fb-8b1f-4417-b735-cd1accb01bc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6802","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6803","nonce":1224390038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6804","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6805","nonce":1224390038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6808","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6809","nonce":1224390038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6806","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6807","nonce":1224390038}]},"public_addr":"10.20.193.12:6803/1224390038","cluster_addr":"10.20.193.12:6805/1224390038","heartbeat_back_addr":"10.20.193.12:6809/1224390038","heartbeat_front_addr":"10.20.193.12:6807/1224390038","state":["exists","up"]},{"osd":2,"uuid":"079aba72-a855-44a4-9127-c34c8eafdc06","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6810","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6811","nonce":1321751267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6812","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6813","nonce":1321751267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6816","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6817","nonce":1321751267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6814","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6815","nonce":1321751267}]},"public_addr":"10.20.193.12:6811/1321751267","cluster_addr":"10.20.193.12:6813/1321751267","heartbeat_back_addr":"10.20.193.12:6817/1321751267","heartbeat_front_addr":"10.20.193.12:6815/1321751267","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:09.575529+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:08.778690+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:09.480453+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:19.431 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-22T19:50:19.432 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-22T19:50:19.433 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-22T19:50:19.515 INFO:teuthology.orchestra.run.trial012.stdout:38654705667 2026-02-22T19:50:19.516 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T19:50:19.517 INFO:teuthology.orchestra.run.trial012.stdout:38654705667 2026-02-22T19:50:19.518 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T19:50:19.520 INFO:teuthology.orchestra.run.trial012.stdout:34359738371 2026-02-22T19:50:19.521 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T19:50:19.674 INFO:teuthology.orchestra.run.trial012.stdout:38654705666 2026-02-22T19:50:19.687 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705666 for osd.2 2026-02-22T19:50:19.708 INFO:teuthology.orchestra.run.trial012.stdout:38654705666 2026-02-22T19:50:19.708 INFO:teuthology.orchestra.run.trial012.stdout:34359738370 2026-02-22T19:50:19.719 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705666 for osd.0 2026-02-22T19:50:19.720 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.1 2026-02-22T19:50:20.688 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T19:50:20.720 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T19:50:20.721 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T19:50:20.841 INFO:teuthology.orchestra.run.trial012.stdout:38654705667 2026-02-22T19:50:20.853 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705667 for osd.2 2026-02-22T19:50:20.854 DEBUG:teuthology.parallel:result is None 2026-02-22T19:50:20.886 INFO:teuthology.orchestra.run.trial012.stdout:38654705667 2026-02-22T19:50:20.890 INFO:teuthology.orchestra.run.trial012.stdout:34359738371 2026-02-22T19:50:20.897 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705667 for osd.0 2026-02-22T19:50:20.897 DEBUG:teuthology.parallel:result is None 2026-02-22T19:50:20.902 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.1 2026-02-22T19:50:20.903 DEBUG:teuthology.parallel:result is None 2026-02-22T19:50:20.903 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T19:50:20.903 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T19:50:21.088 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:21.088 INFO:teuthology.orchestra.run.trial012.stderr:dumped all 2026-02-22T19:50:21.098 INFO:teuthology.orchestra.run.trial012.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-02-22T19:50:19.889191+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":10,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81432,"kb_used_data":536,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283034088,"statfs":{"total":289910292480,"available":289826906112,"internally_reserved":0,"allocated":548864,"data_stored":1302985,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4781,"internal_metadata":82767187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.976337"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919890+0000","last_change":"2026-02-22T19:50:18.919890+0000","last_active":"2026-02-22T19:50:18.919890+0000","last_peered":"2026-02-22T19:50:18.919890+0000","last_clean":"2026-02-22T19:50:18.919890+0000","last_became_active":"2026-02-22T19:50:16.919587+0000","last_became_peered":"2026-02-22T19:50:16.919587+0000","last_unstale":"2026-02-22T19:50:18.919890+0000","last_undegraded":"2026-02-22T19:50:18.919890+0000","last_fullsized":"2026-02-22T19:50:18.919890+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:55:28.698393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389625,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919963+0000","last_change":"2026-02-22T19:50:18.919963+0000","last_active":"2026-02-22T19:50:18.919963+0000","last_peered":"2026-02-22T19:50:18.919963+0000","last_clean":"2026-02-22T19:50:18.919963+0000","last_became_active":"2026-02-22T19:50:16.919517+0000","last_became_peered":"2026-02-22T19:50:16.919517+0000","last_unstale":"2026-02-22T19:50:18.919963+0000","last_undegraded":"2026-02-22T19:50:18.919963+0000","last_fullsized":"2026-02-22T19:50:18.919963+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:48:20.395749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046687900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919761+0000","last_change":"2026-02-22T19:50:18.919761+0000","last_active":"2026-02-22T19:50:18.919761+0000","last_peered":"2026-02-22T19:50:18.919761+0000","last_clean":"2026-02-22T19:50:18.919761+0000","last_became_active":"2026-02-22T19:50:16.918703+0000","last_became_peered":"2026-02-22T19:50:16.918703+0000","last_unstale":"2026-02-22T19:50:18.919761+0000","last_undegraded":"2026-02-22T19:50:18.919761+0000","last_fullsized":"2026-02-22T19:50:18.919761+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:54.246239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027795700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.920056+0000","last_change":"2026-02-22T19:50:18.920056+0000","last_active":"2026-02-22T19:50:18.920056+0000","last_peered":"2026-02-22T19:50:18.920056+0000","last_clean":"2026-02-22T19:50:18.920056+0000","last_became_active":"2026-02-22T19:50:16.919409+0000","last_became_peered":"2026-02-22T19:50:16.919409+0000","last_unstale":"2026-02-22T19:50:18.920056+0000","last_undegraded":"2026-02-22T19:50:18.920056+0000","last_fullsized":"2026-02-22T19:50:18.920056+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:48.980771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053050700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.922506+0000","last_change":"2026-02-22T19:50:18.922506+0000","last_active":"2026-02-22T19:50:18.922506+0000","last_peered":"2026-02-22T19:50:18.922506+0000","last_clean":"2026-02-22T19:50:18.922506+0000","last_became_active":"2026-02-22T19:50:16.918968+0000","last_became_peered":"2026-02-22T19:50:16.918968+0000","last_unstale":"2026-02-22T19:50:18.922506+0000","last_undegraded":"2026-02-22T19:50:18.922506+0000","last_fullsized":"2026-02-22T19:50:18.922506+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:44:51.391464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065213399999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:19.270568+0000","last_change":"2026-02-22T19:50:19.270698+0000","last_active":"2026-02-22T19:50:19.270568+0000","last_peered":"2026-02-22T19:50:19.270568+0000","last_clean":"2026-02-22T19:50:19.270568+0000","last_became_active":"2026-02-22T19:50:16.919079+0000","last_became_peered":"2026-02-22T19:50:16.919079+0000","last_unstale":"2026-02-22T19:50:19.270568+0000","last_undegraded":"2026-02-22T19:50:19.270568+0000","last_fullsized":"2026-02-22T19:50:19.270568+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T20:18:59.837206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028262600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:19.270525+0000","last_change":"2026-02-22T19:50:19.270701+0000","last_active":"2026-02-22T19:50:19.270525+0000","last_peered":"2026-02-22T19:50:19.270525+0000","last_clean":"2026-02-22T19:50:19.270525+0000","last_became_active":"2026-02-22T19:50:16.919023+0000","last_became_peered":"2026-02-22T19:50:16.919023+0000","last_unstale":"2026-02-22T19:50:19.270525+0000","last_undegraded":"2026-02-22T19:50:19.270525+0000","last_fullsized":"2026-02-22T19:50:19.270525+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:29:16.953662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033472299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919746+0000","last_change":"2026-02-22T19:50:18.919746+0000","last_active":"2026-02-22T19:50:18.919746+0000","last_peered":"2026-02-22T19:50:18.919746+0000","last_clean":"2026-02-22T19:50:18.919746+0000","last_became_active":"2026-02-22T19:50:16.918296+0000","last_became_peered":"2026-02-22T19:50:16.918296+0000","last_unstale":"2026-02-22T19:50:18.919746+0000","last_undegraded":"2026-02-22T19:50:18.919746+0000","last_fullsized":"2026-02-22T19:50:18.919746+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:31:42.053515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026312999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'72","reported_seq":109,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919377+0000","last_change":"2026-02-22T19:50:14.235773+0000","last_active":"2026-02-22T19:50:18.919377+0000","last_peered":"2026-02-22T19:50:18.919377+0000","last_clean":"2026-02-22T19:50:18.919377+0000","last_became_active":"2026-02-22T19:50:14.235079+0000","last_became_peered":"2026-02-22T19:50:14.235079+0000","last_unstale":"2026-02-22T19:50:18.919377+0000","last_undegraded":"2026-02-22T19:50:18.919377+0000","last_fullsized":"2026-02-22T19:50:18.919377+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_clean_scrub_stamp":"2026-02-22T19:50:12.905907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:02:57.829873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705667,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27100,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344740,"statfs":{"total":96636764160,"available":96609013760,"internally_reserved":0,"allocated":139264,"data_stored":39554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705667,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27172,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344668,"statfs":{"total":96636764160,"available":96608940032,"internally_reserved":0,"allocated":208896,"data_stored":633501,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27160,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344680,"statfs":{"total":96636764160,"available":96608952320,"internally_reserved":0,"allocated":200704,"data_stored":629930,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T19:50:21.099 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T19:50:21.240 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:21.240 INFO:teuthology.orchestra.run.trial012.stderr:dumped all 2026-02-22T19:50:21.250 INFO:teuthology.orchestra.run.trial012.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-02-22T19:50:19.889191+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":10,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81432,"kb_used_data":536,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283034088,"statfs":{"total":289910292480,"available":289826906112,"internally_reserved":0,"allocated":548864,"data_stored":1302985,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4781,"internal_metadata":82767187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.976337"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919890+0000","last_change":"2026-02-22T19:50:18.919890+0000","last_active":"2026-02-22T19:50:18.919890+0000","last_peered":"2026-02-22T19:50:18.919890+0000","last_clean":"2026-02-22T19:50:18.919890+0000","last_became_active":"2026-02-22T19:50:16.919587+0000","last_became_peered":"2026-02-22T19:50:16.919587+0000","last_unstale":"2026-02-22T19:50:18.919890+0000","last_undegraded":"2026-02-22T19:50:18.919890+0000","last_fullsized":"2026-02-22T19:50:18.919890+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:55:28.698393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389625,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919963+0000","last_change":"2026-02-22T19:50:18.919963+0000","last_active":"2026-02-22T19:50:18.919963+0000","last_peered":"2026-02-22T19:50:18.919963+0000","last_clean":"2026-02-22T19:50:18.919963+0000","last_became_active":"2026-02-22T19:50:16.919517+0000","last_became_peered":"2026-02-22T19:50:16.919517+0000","last_unstale":"2026-02-22T19:50:18.919963+0000","last_undegraded":"2026-02-22T19:50:18.919963+0000","last_fullsized":"2026-02-22T19:50:18.919963+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:48:20.395749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046687900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919761+0000","last_change":"2026-02-22T19:50:18.919761+0000","last_active":"2026-02-22T19:50:18.919761+0000","last_peered":"2026-02-22T19:50:18.919761+0000","last_clean":"2026-02-22T19:50:18.919761+0000","last_became_active":"2026-02-22T19:50:16.918703+0000","last_became_peered":"2026-02-22T19:50:16.918703+0000","last_unstale":"2026-02-22T19:50:18.919761+0000","last_undegraded":"2026-02-22T19:50:18.919761+0000","last_fullsized":"2026-02-22T19:50:18.919761+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:54.246239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027795700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.920056+0000","last_change":"2026-02-22T19:50:18.920056+0000","last_active":"2026-02-22T19:50:18.920056+0000","last_peered":"2026-02-22T19:50:18.920056+0000","last_clean":"2026-02-22T19:50:18.920056+0000","last_became_active":"2026-02-22T19:50:16.919409+0000","last_became_peered":"2026-02-22T19:50:16.919409+0000","last_unstale":"2026-02-22T19:50:18.920056+0000","last_undegraded":"2026-02-22T19:50:18.920056+0000","last_fullsized":"2026-02-22T19:50:18.920056+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:48.980771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053050700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.922506+0000","last_change":"2026-02-22T19:50:18.922506+0000","last_active":"2026-02-22T19:50:18.922506+0000","last_peered":"2026-02-22T19:50:18.922506+0000","last_clean":"2026-02-22T19:50:18.922506+0000","last_became_active":"2026-02-22T19:50:16.918968+0000","last_became_peered":"2026-02-22T19:50:16.918968+0000","last_unstale":"2026-02-22T19:50:18.922506+0000","last_undegraded":"2026-02-22T19:50:18.922506+0000","last_fullsized":"2026-02-22T19:50:18.922506+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:44:51.391464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065213399999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:19.270568+0000","last_change":"2026-02-22T19:50:19.270698+0000","last_active":"2026-02-22T19:50:19.270568+0000","last_peered":"2026-02-22T19:50:19.270568+0000","last_clean":"2026-02-22T19:50:19.270568+0000","last_became_active":"2026-02-22T19:50:16.919079+0000","last_became_peered":"2026-02-22T19:50:16.919079+0000","last_unstale":"2026-02-22T19:50:19.270568+0000","last_undegraded":"2026-02-22T19:50:19.270568+0000","last_fullsized":"2026-02-22T19:50:19.270568+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T20:18:59.837206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028262600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:19.270525+0000","last_change":"2026-02-22T19:50:19.270701+0000","last_active":"2026-02-22T19:50:19.270525+0000","last_peered":"2026-02-22T19:50:19.270525+0000","last_clean":"2026-02-22T19:50:19.270525+0000","last_became_active":"2026-02-22T19:50:16.919023+0000","last_became_peered":"2026-02-22T19:50:16.919023+0000","last_unstale":"2026-02-22T19:50:19.270525+0000","last_undegraded":"2026-02-22T19:50:19.270525+0000","last_fullsized":"2026-02-22T19:50:19.270525+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:29:16.953662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033472299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919746+0000","last_change":"2026-02-22T19:50:18.919746+0000","last_active":"2026-02-22T19:50:18.919746+0000","last_peered":"2026-02-22T19:50:18.919746+0000","last_clean":"2026-02-22T19:50:18.919746+0000","last_became_active":"2026-02-22T19:50:16.918296+0000","last_became_peered":"2026-02-22T19:50:16.918296+0000","last_unstale":"2026-02-22T19:50:18.919746+0000","last_undegraded":"2026-02-22T19:50:18.919746+0000","last_fullsized":"2026-02-22T19:50:18.919746+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:31:42.053515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026312999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'72","reported_seq":109,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T19:50:18.919377+0000","last_change":"2026-02-22T19:50:14.235773+0000","last_active":"2026-02-22T19:50:18.919377+0000","last_peered":"2026-02-22T19:50:18.919377+0000","last_clean":"2026-02-22T19:50:18.919377+0000","last_became_active":"2026-02-22T19:50:14.235079+0000","last_became_peered":"2026-02-22T19:50:14.235079+0000","last_unstale":"2026-02-22T19:50:18.919377+0000","last_undegraded":"2026-02-22T19:50:18.919377+0000","last_fullsized":"2026-02-22T19:50:18.919377+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_clean_scrub_stamp":"2026-02-22T19:50:12.905907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:02:57.829873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705667,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27100,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344740,"statfs":{"total":96636764160,"available":96609013760,"internally_reserved":0,"allocated":139264,"data_stored":39554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705667,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27172,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344668,"statfs":{"total":96636764160,"available":96608940032,"internally_reserved":0,"allocated":208896,"data_stored":633501,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27160,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344680,"statfs":{"total":96636764160,"available":96608952320,"internally_reserved":0,"allocated":200704,"data_stored":629930,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T19:50:21.251 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T19:50:21.251 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-22T19:50:21.251 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-22T19:50:21.252 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-22T19:50:21.408 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:21.409 INFO:teuthology.orchestra.run.trial012.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T19:50:21.420 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-22T19:50:21.420 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T19:50:21.435 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-22T19:50:21.435 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zstd', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T19:50:21.435 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zstd supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-workunit-minimum}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64002', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zstd', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial012.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHr8AQoPmLBCDlwyLfkwSBORD6UrQX2i0NkRJcfXP05UjfcnISwAcRU/7PWmtbs6gaA2+U8uoxsT8lx4AB9jXYs=', 'trial098.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFhjjShomKSln4JXn92gq3TmrFoct33rjWGfkyU/WvmlES3UzQ3qMzMbVCSvAeKL2FJLfN0ObmSkRB/x1W++kek='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zstd', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zstd', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_IMAGE_FEATURES': 'layering', 'RBD_MIRROR_CONFIG_KEY': '1', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T19:50:21.436 DEBUG:teuthology.orchestra.run.trial012:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T19:50:21.438 DEBUG:teuthology.orchestra.run.trial098:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T19:50:21.444 DEBUG:teuthology.orchestra.run.trial012:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T19:50:21.486 DEBUG:teuthology.orchestra.run.trial098:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T19:50:21.495 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:21.495 DEBUG:teuthology.orchestra.run.trial098:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T19:50:21.539 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T19:50:21.539 DEBUG:teuthology.orchestra.run.trial098:> stat /dev/vg_nvme/lv_1 2026-02-22T19:50:21.586 INFO:teuthology.orchestra.run.trial098.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T19:50:21.587 INFO:teuthology.orchestra.run.trial098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T19:50:21.587 INFO:teuthology.orchestra.run.trial098.stdout:Device: 5h/5d Inode: 905 Links: 1 2026-02-22T19:50:21.587 INFO:teuthology.orchestra.run.trial098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T19:50:21.587 INFO:teuthology.orchestra.run.trial098.stdout:Access: 2026-02-22 19:45:49.820303992 +0000 2026-02-22T19:50:21.587 INFO:teuthology.orchestra.run.trial098.stdout:Modify: 2026-02-22 19:45:49.624307323 +0000 2026-02-22T19:50:21.587 INFO:teuthology.orchestra.run.trial098.stdout:Change: 2026-02-22 19:45:49.624307323 +0000 2026-02-22T19:50:21.587 INFO:teuthology.orchestra.run.trial098.stdout: Birth: - 2026-02-22T19:50:21.587 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T19:50:21.633 INFO:teuthology.orchestra.run.trial098.stderr:1+0 records in 2026-02-22T19:50:21.633 INFO:teuthology.orchestra.run.trial098.stderr:1+0 records out 2026-02-22T19:50:21.633 INFO:teuthology.orchestra.run.trial098.stderr:512 bytes copied, 0.000192569 s, 2.7 MB/s 2026-02-22T19:50:21.634 DEBUG:teuthology.orchestra.run.trial098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T19:50:21.680 DEBUG:teuthology.orchestra.run.trial098:> stat /dev/vg_nvme/lv_2 2026-02-22T19:50:21.726 INFO:teuthology.orchestra.run.trial098.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T19:50:21.727 INFO:teuthology.orchestra.run.trial098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T19:50:21.727 INFO:teuthology.orchestra.run.trial098.stdout:Device: 5h/5d Inode: 941 Links: 1 2026-02-22T19:50:21.727 INFO:teuthology.orchestra.run.trial098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T19:50:21.727 INFO:teuthology.orchestra.run.trial098.stdout:Access: 2026-02-22 19:45:50.004300862 +0000 2026-02-22T19:50:21.727 INFO:teuthology.orchestra.run.trial098.stdout:Modify: 2026-02-22 19:45:50.000300932 +0000 2026-02-22T19:50:21.727 INFO:teuthology.orchestra.run.trial098.stdout:Change: 2026-02-22 19:45:50.000300932 +0000 2026-02-22T19:50:21.727 INFO:teuthology.orchestra.run.trial098.stdout: Birth: - 2026-02-22T19:50:21.728 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T19:50:21.773 INFO:teuthology.orchestra.run.trial098.stderr:1+0 records in 2026-02-22T19:50:21.774 INFO:teuthology.orchestra.run.trial098.stderr:1+0 records out 2026-02-22T19:50:21.774 INFO:teuthology.orchestra.run.trial098.stderr:512 bytes copied, 0.000172111 s, 3.0 MB/s 2026-02-22T19:50:21.775 DEBUG:teuthology.orchestra.run.trial098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T19:50:21.824 DEBUG:teuthology.orchestra.run.trial098:> stat /dev/vg_nvme/lv_3 2026-02-22T19:50:21.870 INFO:teuthology.orchestra.run.trial098.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T19:50:21.871 INFO:teuthology.orchestra.run.trial098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T19:50:21.871 INFO:teuthology.orchestra.run.trial098.stdout:Device: 5h/5d Inode: 953 Links: 1 2026-02-22T19:50:21.871 INFO:teuthology.orchestra.run.trial098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T19:50:21.871 INFO:teuthology.orchestra.run.trial098.stdout:Access: 2026-02-22 19:45:50.308295697 +0000 2026-02-22T19:50:21.871 INFO:teuthology.orchestra.run.trial098.stdout:Modify: 2026-02-22 19:45:50.304295762 +0000 2026-02-22T19:50:21.871 INFO:teuthology.orchestra.run.trial098.stdout:Change: 2026-02-22 19:45:50.304295762 +0000 2026-02-22T19:50:21.871 INFO:teuthology.orchestra.run.trial098.stdout: Birth: - 2026-02-22T19:50:21.872 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T19:50:21.921 INFO:teuthology.orchestra.run.trial098.stderr:1+0 records in 2026-02-22T19:50:21.921 INFO:teuthology.orchestra.run.trial098.stderr:1+0 records out 2026-02-22T19:50:21.921 INFO:teuthology.orchestra.run.trial098.stderr:512 bytes copied, 0.000188471 s, 2.7 MB/s 2026-02-22T19:50:21.922 DEBUG:teuthology.orchestra.run.trial098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T19:50:21.968 DEBUG:teuthology.orchestra.run.trial098:> stat /dev/vg_nvme/lv_4 2026-02-22T19:50:22.014 INFO:teuthology.orchestra.run.trial098.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T19:50:22.014 INFO:teuthology.orchestra.run.trial098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T19:50:22.015 INFO:teuthology.orchestra.run.trial098.stdout:Device: 5h/5d Inode: 979 Links: 1 2026-02-22T19:50:22.015 INFO:teuthology.orchestra.run.trial098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T19:50:22.015 INFO:teuthology.orchestra.run.trial098.stdout:Access: 2026-02-22 19:45:50.632290190 +0000 2026-02-22T19:50:22.015 INFO:teuthology.orchestra.run.trial098.stdout:Modify: 2026-02-22 19:45:50.628290256 +0000 2026-02-22T19:50:22.015 INFO:teuthology.orchestra.run.trial098.stdout:Change: 2026-02-22 19:45:50.628290256 +0000 2026-02-22T19:50:22.015 INFO:teuthology.orchestra.run.trial098.stdout: Birth: - 2026-02-22T19:50:22.015 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T19:50:22.062 INFO:teuthology.orchestra.run.trial098.stderr:1+0 records in 2026-02-22T19:50:22.062 INFO:teuthology.orchestra.run.trial098.stderr:1+0 records out 2026-02-22T19:50:22.062 INFO:teuthology.orchestra.run.trial098.stderr:512 bytes copied, 0.00017687 s, 2.9 MB/s 2026-02-22T19:50:22.063 DEBUG:teuthology.orchestra.run.trial098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T19:50:22.112 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T19:50:22.112 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial098.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T19:50:22.112 INFO:tasks.ceph:Generating config... 2026-02-22T19:50:22.114 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.115 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.116 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.116 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.116 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.116 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.116 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.116 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.116 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T19:50:22.116 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T19:50:22.116 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T19:50:22.116 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T19:50:22.116 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T19:50:22.116 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T19:50:22.117 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T19:50:22.117 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] bluestore compression algorithm = zstd 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T19:50:22.117 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T19:50:22.118 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T19:50:22.118 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T19:50:22.118 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T19:50:22.118 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T19:50:22.118 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-22T19:50:22.118 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-22T19:50:22.171 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster2.keyring 2026-02-22T19:50:22.173 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-22T19:50:22.233 DEBUG:teuthology.orchestra.run.trial098:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T19:50:22.282 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.98')] 2026-02-22T19:50:22.283 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.98', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zstd', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T19:50:22.284 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:22.284 DEBUG:teuthology.orchestra.run.trial098:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T19:50:22.327 DEBUG:teuthology.orchestra.run.trial098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.98 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T19:50:22.383 INFO:teuthology.orchestra.run.trial098.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T19:50:22.383 INFO:teuthology.orchestra.run.trial098.stdout:monmaptool: generated fsid c78b27d3-80f9-4252-ae60-d838c71871f8 2026-02-22T19:50:22.383 INFO:teuthology.orchestra.run.trial098.stdout:setting min_mon_release = quincy 2026-02-22T19:50:22.383 INFO:teuthology.orchestra.run.trial098.stdout:epoch 0 2026-02-22T19:50:22.384 INFO:teuthology.orchestra.run.trial098.stdout:fsid c78b27d3-80f9-4252-ae60-d838c71871f8 2026-02-22T19:50:22.384 INFO:teuthology.orchestra.run.trial098.stdout:last_changed 2026-02-22T19:50:22.384817+0000 2026-02-22T19:50:22.384 INFO:teuthology.orchestra.run.trial098.stdout:created 2026-02-22T19:50:22.384817+0000 2026-02-22T19:50:22.384 INFO:teuthology.orchestra.run.trial098.stdout:min_mon_release 17 (quincy) 2026-02-22T19:50:22.384 INFO:teuthology.orchestra.run.trial098.stdout:election_strategy: 1 2026-02-22T19:50:22.384 INFO:teuthology.orchestra.run.trial098.stdout:0: [v2:10.20.193.98:3300/0,v1:10.20.193.98:6789/0] mon.a 2026-02-22T19:50:22.384 INFO:teuthology.orchestra.run.trial098.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-22T19:50:22.385 DEBUG:teuthology.orchestra.run.trial098:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T19:50:22.431 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID c78b27d3-80f9-4252-ae60-d838c71871f8... 2026-02-22T19:50:22.432 DEBUG:teuthology.orchestra.run.trial012:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T19:50:22.434 DEBUG:teuthology.orchestra.run.trial098:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T19:50:22.473 INFO:teuthology.orchestra.run.trial012.stdout:[global] 2026-02-22T19:50:22.473 INFO:teuthology.orchestra.run.trial012.stdout: chdir = "" 2026-02-22T19:50:22.473 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T19:50:22.473 INFO:teuthology.orchestra.run.trial012.stdout: auth supported = cephx 2026-02-22T19:50:22.473 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.473 INFO:teuthology.orchestra.run.trial012.stdout: filestore xattr use omap = true 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: mon clock drift allowed = 1.000 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: osd crush chooseleaf type = 0 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: auth debug = true 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: ms die on old message = true 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: ms die on bug = true 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T19:50:22.474 INFO:teuthology.orchestra.run.trial012.stdout: mon pg warn max object skew = 0 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: osd pool default size = 2 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: mon osd allow primary affinity = true 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: mon osd allow pg remap = true 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on legacy crush tunables = false 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on crush straw calc version zero = false 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on no sortbitwise = false 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on osd down out interval zero = false 2026-02-22T19:50:22.475 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on too few osds = false 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: mon_allow_pool_size_one = true 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: osd default data pool replay window = 5 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: mon allow pool delete = true 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: mon cluster log file level = debug 2026-02-22T19:50:22.476 INFO:teuthology.orchestra.run.trial012.stdout: debug asserts on shutdown = true 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: mon health detail to clog = false 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: mon host = 10.20.193.98 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: mon client directed command retry = 5 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: mon warn on pool no app = False 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: ms inject socket failures = 5000 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: fsid = c78b27d3-80f9-4252-ae60-d838c71871f8 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout:[osd] 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: osd journal size = 100 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: osd scrub load threshold = 5.0 2026-02-22T19:50:22.477 INFO:teuthology.orchestra.run.trial012.stdout: osd scrub max interval = 600 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: osd mclock profile = high_recovery_ops 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: osd mclock skip benchmark = true 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: osd recover clone overlap = true 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: osd recovery max chunk = 1048576 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: osd debug shutdown = true 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: osd debug op order = true 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: osd debug verify stray on activate = true 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: osd debug trim objects = true 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.478 INFO:teuthology.orchestra.run.trial012.stdout: osd open classes on start = true 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: osd debug pg log writeout = true 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: osd deep scrub update digest min age = 30 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: osd map max advance = 10 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: journal zero on create = true 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: filestore ondisk finisher threads = 3 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: filestore apply finisher threads = 3 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: bdev debug aio = true 2026-02-22T19:50:22.479 INFO:teuthology.orchestra.run.trial012.stdout: osd debug misdirected ops = true 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: bluestore block size = 96636764160 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: bluestore compression algorithm = zstd 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: bluestore compression mode = aggressive 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: bluestore fsck on mount = True 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: debug bluefs = 1/20 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: debug bluestore = 1/20 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: debug ms = 1 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: debug osd = 20 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: debug rocksdb = 4/10 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: mon osd full ratio = 0.9 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: mon osd nearfull ratio = 0.8 2026-02-22T19:50:22.480 INFO:teuthology.orchestra.run.trial012.stdout: osd failsafe full ratio = 0.95 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: osd objectstore = bluestore 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout:[mgr] 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: debug ms = 1 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: debug mgr = 20 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: debug mon = 20 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: debug auth = 20 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: mon reweight min pgs per osd = 4 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: mon reweight min bytes per osd = 10 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: mgr/telemetry/nag = false 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.481 INFO:teuthology.orchestra.run.trial012.stdout:[mon] 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: debug ms = 1 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: debug mon = 20 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: debug paxos = 20 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: debug auth = 20 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: mon data avail warn = 5 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: mon mgr mkfs grace = 240 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: mon reweight min pgs per osd = 4 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: mon osd reporter subtree level = osd 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: mon osd prime pg temp = true 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: mon reweight min bytes per osd = 10 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T19:50:22.482 INFO:teuthology.orchestra.run.trial012.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: # 1m isn't quite enough 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: mon_down_mkfs_grace = 2m 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: mon_warn_on_filestore_osds = false 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout:[client] 2026-02-22T19:50:22.483 INFO:teuthology.orchestra.run.trial012.stdout: rgw cache enabled = true 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: rgw enable ops log = true 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: rgw enable usage log = true 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: debug rbd = 20 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: debug rbd_mirror = 15 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: log to stderr = False 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: rbd default features = 125 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout:[mon.a] 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.0] 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.484 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.1] 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.2] 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.3] 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.4] 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.5] 2026-02-22T19:50:22.485 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.486 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.486 INFO:teuthology.orchestra.run.trial012.stdout:[client.mirror.6] 2026-02-22T19:50:22.486 INFO:teuthology.orchestra.run.trial012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.486 INFO:teuthology.orchestra.run.trial012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout:[global] 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout: chdir = "" 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout: auth supported = cephx 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout: filestore xattr use omap = true 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout: mon clock drift allowed = 1.000 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.490 INFO:teuthology.orchestra.run.trial098.stdout: osd crush chooseleaf type = 0 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: auth debug = true 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: ms die on old message = true 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: ms die on bug = true 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: mon pg warn max object skew = 0 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T19:50:22.491 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: osd pool default size = 2 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon osd allow primary affinity = true 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon osd allow pg remap = true 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on legacy crush tunables = false 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on crush straw calc version zero = false 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on no sortbitwise = false 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on osd down out interval zero = false 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on too few osds = false 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: mon_allow_pool_size_one = true 2026-02-22T19:50:22.492 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: osd default data pool replay window = 5 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: mon allow pool delete = true 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: mon cluster log file level = debug 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: debug asserts on shutdown = true 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: mon health detail to clog = false 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: mon host = 10.20.193.98 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: mon client directed command retry = 5 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: mon warn on pool no app = False 2026-02-22T19:50:22.493 INFO:teuthology.orchestra.run.trial098.stdout: ms inject socket failures = 5000 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: fsid = c78b27d3-80f9-4252-ae60-d838c71871f8 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout:[osd] 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: osd journal size = 100 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: osd scrub load threshold = 5.0 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: osd scrub max interval = 600 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: osd mclock profile = high_recovery_ops 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: osd mclock skip benchmark = true 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: osd recover clone overlap = true 2026-02-22T19:50:22.494 INFO:teuthology.orchestra.run.trial098.stdout: osd recovery max chunk = 1048576 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: osd debug shutdown = true 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: osd debug op order = true 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: osd debug verify stray on activate = true 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: osd debug trim objects = true 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: osd open classes on start = true 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: osd debug pg log writeout = true 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: osd deep scrub update digest min age = 30 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: osd map max advance = 10 2026-02-22T19:50:22.495 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: journal zero on create = true 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: filestore ondisk finisher threads = 3 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: filestore apply finisher threads = 3 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: bdev debug aio = true 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: osd debug misdirected ops = true 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: bluestore block size = 96636764160 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: bluestore compression algorithm = zstd 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: bluestore compression mode = aggressive 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: bluestore fsck on mount = True 2026-02-22T19:50:22.496 INFO:teuthology.orchestra.run.trial098.stdout: debug bluefs = 1/20 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: debug bluestore = 1/20 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: debug ms = 1 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: debug osd = 20 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: debug rocksdb = 4/10 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: mon osd full ratio = 0.9 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: mon osd nearfull ratio = 0.8 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: osd failsafe full ratio = 0.95 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: osd objectstore = bluestore 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout:[mgr] 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: debug ms = 1 2026-02-22T19:50:22.497 INFO:teuthology.orchestra.run.trial098.stdout: debug mgr = 20 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: debug mon = 20 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: debug auth = 20 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: mon reweight min pgs per osd = 4 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: mon reweight min bytes per osd = 10 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: mgr/telemetry/nag = false 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout:[mon] 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: debug ms = 1 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: debug mon = 20 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: debug paxos = 20 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: debug auth = 20 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: mon data avail warn = 5 2026-02-22T19:50:22.498 INFO:teuthology.orchestra.run.trial098.stdout: mon mgr mkfs grace = 240 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: mon reweight min pgs per osd = 4 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: mon osd reporter subtree level = osd 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: mon osd prime pg temp = true 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: mon reweight min bytes per osd = 10 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T19:50:22.499 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: # 1m isn't quite enough 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: mon_down_mkfs_grace = 2m 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: mon_warn_on_filestore_osds = false 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout:[client] 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: rgw cache enabled = true 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: rgw enable ops log = true 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: rgw enable usage log = true 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: debug rbd = 20 2026-02-22T19:50:22.500 INFO:teuthology.orchestra.run.trial098.stdout: debug rbd_mirror = 15 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout: log to stderr = False 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout: rbd default features = 125 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout:[mon.a] 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.0] 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.1] 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.2] 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.501 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.3] 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.4] 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.5] 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.6] 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T19:50:22.502 INFO:teuthology.orchestra.run.trial098.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T19:50:22.503 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-22T19:50:22.504 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-22T19:50:22.568 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T19:50:22.569 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:22.569 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-22T19:50:22.615 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:22.615 DEBUG:teuthology.orchestra.run.trial098:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-22T19:50:22.663 INFO:tasks.ceph:Sending monmap to node ubuntu@trial012.front.sepia.ceph.com 2026-02-22T19:50:22.663 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:22.663 DEBUG:teuthology.orchestra.run.trial012:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T19:50:22.663 DEBUG:teuthology.orchestra.run.trial012:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T19:50:22.677 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:22.677 DEBUG:teuthology.orchestra.run.trial012:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T19:50:22.724 INFO:tasks.ceph:Sending monmap to node ubuntu@trial098.front.sepia.ceph.com 2026-02-22T19:50:22.725 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:22.725 DEBUG:teuthology.orchestra.run.trial098:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T19:50:22.725 DEBUG:teuthology.orchestra.run.trial098:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T19:50:22.740 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:22.740 DEBUG:teuthology.orchestra.run.trial098:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T19:50:22.787 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T19:50:22.787 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T19:50:22.787 DEBUG:teuthology.orchestra.run.trial012:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T19:50:22.810 INFO:teuthology.orchestra.run.trial012.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T19:50:22.812 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T19:50:22.813 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T19:50:22.813 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-22T19:50:22.869 INFO:teuthology.orchestra.run.trial012.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-22T19:50:22.875 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T19:50:22.892 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T19:50:22.899 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T19:50:22.959 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T19:50:22.967 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T19:50:23.028 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T19:50:23.036 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T19:50:23.094 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T19:50:23.102 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T19:50:23.159 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T19:50:23.166 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T19:50:23.228 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T19:50:23.236 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T19:50:23.301 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T19:50:23.308 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T19:50:23.373 INFO:teuthology.orchestra.run.trial098.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T19:50:23.382 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T19:50:23.382 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial012.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial098.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T19:50:23.383 DEBUG:teuthology.orchestra.run.trial098:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-22T19:50:23.431 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T19:50:23.431 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-22T19:50:23.431 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial098.front.sepia.ceph.com 2026-02-22T19:50:23.431 DEBUG:teuthology.orchestra.run.trial098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T19:50:23.484 INFO:teuthology.orchestra.run.trial098.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T19:50:23.485 INFO:teuthology.orchestra.run.trial098.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T19:50:23.485 INFO:teuthology.orchestra.run.trial098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T19:50:23.485 INFO:teuthology.orchestra.run.trial098.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T19:50:23.485 INFO:teuthology.orchestra.run.trial098.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T19:50:23.485 INFO:teuthology.orchestra.run.trial098.stdout: = sunit=0 swidth=0 blks 2026-02-22T19:50:23.485 INFO:teuthology.orchestra.run.trial098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T19:50:23.485 INFO:teuthology.orchestra.run.trial098.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T19:50:23.485 INFO:teuthology.orchestra.run.trial098.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T19:50:23.485 INFO:teuthology.orchestra.run.trial098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T19:50:24.060 INFO:teuthology.orchestra.run.trial098.stdout:Discarding blocks...Done. 2026-02-22T19:50:24.062 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial098.front.sepia.ceph.com -o noatime 2026-02-22T19:50:24.062 DEBUG:teuthology.orchestra.run.trial098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-22T19:50:24.115 DEBUG:teuthology.orchestra.run.trial098:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-22T19:50:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T19:50:24.165 INFO:teuthology.orchestra.run.trial098.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T19:50:24.165 DEBUG:teuthology.orchestra.run.trial098:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-22T19:50:24.215 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T19:50:24.215 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-22T19:50:24.215 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial098.front.sepia.ceph.com 2026-02-22T19:50:24.215 DEBUG:teuthology.orchestra.run.trial098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T19:50:24.268 INFO:teuthology.orchestra.run.trial098.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T19:50:24.268 INFO:teuthology.orchestra.run.trial098.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T19:50:24.268 INFO:teuthology.orchestra.run.trial098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T19:50:24.268 INFO:teuthology.orchestra.run.trial098.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T19:50:24.269 INFO:teuthology.orchestra.run.trial098.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T19:50:24.269 INFO:teuthology.orchestra.run.trial098.stdout: = sunit=0 swidth=0 blks 2026-02-22T19:50:24.269 INFO:teuthology.orchestra.run.trial098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T19:50:24.269 INFO:teuthology.orchestra.run.trial098.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T19:50:24.269 INFO:teuthology.orchestra.run.trial098.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T19:50:24.269 INFO:teuthology.orchestra.run.trial098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T19:50:24.833 INFO:teuthology.orchestra.run.trial098.stdout:Discarding blocks...Done. 2026-02-22T19:50:24.835 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial098.front.sepia.ceph.com -o noatime 2026-02-22T19:50:24.835 DEBUG:teuthology.orchestra.run.trial098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-22T19:50:24.892 DEBUG:teuthology.orchestra.run.trial098:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-22T19:50:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T19:50:24.941 INFO:teuthology.orchestra.run.trial098.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T19:50:24.941 DEBUG:teuthology.orchestra.run.trial098:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-22T19:50:24.991 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T19:50:24.991 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-22T19:50:24.991 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial098.front.sepia.ceph.com 2026-02-22T19:50:24.991 DEBUG:teuthology.orchestra.run.trial098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout: = sunit=0 swidth=0 blks 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T19:50:25.045 INFO:teuthology.orchestra.run.trial098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T19:50:25.609 INFO:teuthology.orchestra.run.trial098.stdout:Discarding blocks...Done. 2026-02-22T19:50:25.611 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial098.front.sepia.ceph.com -o noatime 2026-02-22T19:50:25.611 DEBUG:teuthology.orchestra.run.trial098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-22T19:50:25.673 DEBUG:teuthology.orchestra.run.trial098:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-22T19:50:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T19:50:25.725 INFO:teuthology.orchestra.run.trial098.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T19:50:25.726 DEBUG:teuthology.orchestra.run.trial098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T19:50:25.787 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:25.787+0000 7ff2c5d5a8c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-22T19:50:25.788 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:25.787+0000 7ff2c5d5a8c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-22T19:50:25.788 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:25.787+0000 7ff2c5d5a8c0 -1 bdev(0x559c35b8b800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-22T19:50:25.788 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:25.787+0000 7ff2c5d5a8c0 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-22T19:50:28.911 DEBUG:teuthology.orchestra.run.trial098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-22T19:50:28.964 DEBUG:teuthology.orchestra.run.trial098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T19:50:29.027 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:29.027+0000 7f2187bab8c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-22T19:50:29.027 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:29.027+0000 7f2187bab8c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-22T19:50:29.027 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:29.027+0000 7f2187bab8c0 -1 bdev(0x56369ed27800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-22T19:50:29.027 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:29.027+0000 7f2187bab8c0 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-22T19:50:32.155 DEBUG:teuthology.orchestra.run.trial098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-22T19:50:32.208 DEBUG:teuthology.orchestra.run.trial098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T19:50:32.272 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:32.271+0000 7f7a4f0228c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-22T19:50:32.273 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:32.271+0000 7f7a4f0228c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-22T19:50:32.273 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:32.271+0000 7f7a4f0228c0 -1 bdev(0x55ed690e9800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-22T19:50:32.273 INFO:teuthology.orchestra.run.trial098.stderr:2026-02-22T19:50:32.271+0000 7f7a4f0228c0 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-22T19:50:35.366 DEBUG:teuthology.orchestra.run.trial098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-22T19:50:35.416 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T19:50:35.416 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:35.416 DEBUG:teuthology.orchestra.run.trial012:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-22T19:50:35.427 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.427 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-22T19:50:35.468 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.468 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-22T19:50:35.520 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.520 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-22T19:50:35.572 DEBUG:teuthology.orchestra.run.trial012:> set -ex 2026-02-22T19:50:35.572 DEBUG:teuthology.orchestra.run.trial012:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-22T19:50:35.578 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.579 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-22T19:50:35.619 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.619 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-22T19:50:35.667 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.667 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-22T19:50:35.711 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.711 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-22T19:50:35.758 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.759 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-22T19:50:35.802 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.803 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-22T19:50:35.847 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.847 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-22T19:50:35.894 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:35.895 DEBUG:teuthology.orchestra.run.trial098:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-22T19:50:35.939 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T19:50:35.939 DEBUG:teuthology.orchestra.run.trial098:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-22T19:50:35.989 INFO:teuthology.orchestra.run.trial098.stdout:[mgr.x] 2026-02-22T19:50:35.989 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD+XZtp3whGMBAAoWwizbImtyNBY4x1T6IQeQ== 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout:[osd.0] 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout: key = AQABXptpQ9EILxAAVl2meFusVyJDrJQM4U37xw== 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout:[osd.1] 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout: key = AQAFXptpN2O+ARAAF7XFrX2fAxLzqj8bZI7xow== 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout:[osd.2] 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout: key = AQAIXptpeh9VEBAAlczGLELmxMXfTJK/nHiBDw== 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout:[client.0] 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD+XZtpTbnFMxAAr6ApIfLiXSMn7tuvp1m8fw== 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror] 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD+XZtprn9PNRAA2CzkRwyKAn5PTTq28VFsOw== 2026-02-22T19:50:35.990 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.0] 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD+XZtpVsBPORAAKyxXRz4sKTMFqOx0koobxA== 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.1] 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpbmTMARAA0la633W88D+eYp9UOAw5Qg== 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.2] 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpZyPBBRAACdv2YW5tTz1dmTF/8Nxqhw== 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.3] 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtp++WVCRAAhcsl0QepXe2Q/mHbJKs+Og== 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.4] 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpGKS7DRAAlT/BeRWIyejFbzhK6SHEKg== 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.5] 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpPJgKEhAAIiCHZhxJhwiIB0uwwNo/ag== 2026-02-22T19:50:35.991 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.6] 2026-02-22T19:50:35.992 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpIahWFhAAZke4pzDPATr7dQdTV5Eqnw== 2026-02-22T19:50:35.993 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T19:50:36.057 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T19:50:36.121 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T19:50:36.183 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T19:50:36.247 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:36.312 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:36.376 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:36.440 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:36.504 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:36.568 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:36.632 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:36.696 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:36.760 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T19:50:36.825 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T19:50:36.826 DEBUG:teuthology.orchestra.run.trial098:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-22T19:50:36.874 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-22T19:50:36.944 DEBUG:teuthology.orchestra.run.trial098:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-22T19:50:36.995 DEBUG:teuthology.orchestra.run.trial098:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T19:50:37.038 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-22T19:50:37.039 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-22T19:50:37.039 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-22T19:50:37.080 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-22T19:50:37.080 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-22T19:50:37.080 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-22T19:50:37.080 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-22T19:50:37.083 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-22T19:50:37.083 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T19:50:37.083 DEBUG:teuthology.orchestra.run.trial098:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-22T19:50:37.203 INFO:teuthology.orchestra.run.trial098.stderr:adjusted tunables profile to default 2026-02-22T19:50:37.213 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T19:50:37.213 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-22T19:50:37.213 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:37.214 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-22T19:50:37.223 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:37.223 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-22T19:50:37.224 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:37.216+0000 7fc23bfa3280 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T19:50:37.249 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:37.244+0000 7fc23bfa3280 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T19:50:37.275 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:50:37.276 DEBUG:teuthology.orchestra.run.trial098:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-22T19:50:37.285 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:37.280+0000 7fc23bfa3280 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T19:50:37.327 DEBUG:teuthology.orchestra.run.trial098:> sudo ceph --cluster cluster2 osd new 0b31b72f-873e-4231-92d2-2bf473c7fb49 0 2026-02-22T19:50:37.350 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:37.344+0000 7fc23bfa3280 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T19:50:37.417 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:37.412+0000 7fc23bfa3280 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T19:50:37.454 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:37.448+0000 7fc23bfa3280 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T19:50:37.468 INFO:teuthology.orchestra.run.trial098.stdout:0 2026-02-22T19:50:37.477 DEBUG:teuthology.orchestra.run.trial098:> sudo ceph --cluster cluster2 osd new 12f18cab-1f27-4a1f-9342-68723365376d 1 2026-02-22T19:50:37.523 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:37.516+0000 7fc23bfa3280 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T19:50:37.572 INFO:teuthology.orchestra.run.trial098.stdout:1 2026-02-22T19:50:37.581 DEBUG:teuthology.orchestra.run.trial098:> sudo ceph --cluster cluster2 osd new 96497b35-d35f-466d-a113-5b0925d9f0c8 2 2026-02-22T19:50:37.677 INFO:teuthology.orchestra.run.trial098.stdout:2 2026-02-22T19:50:37.686 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-22T19:50:37.687 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-22T19:50:37.688 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:37.680+0000 7fc23bfa3280 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T19:50:37.689 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-22T19:50:37.689 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-22T19:50:37.689 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-22T19:50:37.691 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-22T19:50:37.691 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-22T19:50:37.691 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-22T19:50:37.693 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-22T19:50:37.694 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T19:50:37.797 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:37.797 INFO:teuthology.orchestra.run.trial098.stdout:{"epoch":5,"fsid":"c78b27d3-80f9-4252-ae60-d838c71871f8","created":"2026-02-22T19:50:37.162442+0000","modified":"2026-02-22T19:50:37.678649+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T19:50:37.678649+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"0b31b72f-873e-4231-92d2-2bf473c7fb49","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"12f18cab-1f27-4a1f-9342-68723365376d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"96497b35-d35f-466d-a113-5b0925d9f0c8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:37.807 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-22T19:50:37.807 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T19:50:37.926 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:37.920+0000 7fc23bfa3280 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T19:50:38.013 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.008+0000 7fc23bfa3280 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T19:50:38.038 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.032+0000 7fc23bfa3280 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T19:50:38.065 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.060+0000 7fc23bfa3280 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T19:50:38.095 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.088+0000 7fc23bfa3280 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T19:50:38.109 DEBUG:teuthology.orchestra.run.trial098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T19:50:38.187 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.180+0000 7fc23bfa3280 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T19:50:38.200 INFO:teuthology.misc.health.trial098.stdout: 2026-02-22T19:50:38.200 INFO:teuthology.misc.health.trial098.stdout:{"epoch":5,"fsid":"c78b27d3-80f9-4252-ae60-d838c71871f8","created":"2026-02-22T19:50:37.162442+0000","modified":"2026-02-22T19:50:37.678649+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T19:50:37.678649+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"0b31b72f-873e-4231-92d2-2bf473c7fb49","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"12f18cab-1f27-4a1f-9342-68723365376d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"96497b35-d35f-466d-a113-5b0925d9f0c8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:38.209 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T19:50:38.350 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-22T19:50:38.494 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.488+0000 7fc23bfa3280 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T19:50:38.525 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.520+0000 7fc23bfa3280 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T19:50:38.594 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.588+0000 7fc23bfa3280 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T19:50:38.619 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.612+0000 7fc23bfa3280 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T19:50:38.643 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.636+0000 7fc23bfa3280 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T19:50:38.668 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.660+0000 7fc23bfa3280 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T19:50:38.744 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T19:50:38.744 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T19:50:38.744 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr: from numpy import show_config as show_numpy_config 2026-02-22T19:50:38.750 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.744+0000 7fc23bfa3280 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T19:50:38.783 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.776+0000 7fc23bfa3280 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T19:50:38.819 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:38.812+0000 7fc23bfa3280 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T19:50:39.240 INFO:tasks.ceph.cluster2.osd.0.trial098.stderr:2026-02-22T19:50:39.239+0000 7f21abd3b8c0 -1 Falling back to public interface 2026-02-22T19:50:39.241 INFO:tasks.ceph.cluster2.osd.2.trial098.stderr:2026-02-22T19:50:39.239+0000 7f3ec19ac8c0 -1 Falling back to public interface 2026-02-22T19:50:39.448 INFO:tasks.ceph.cluster2.osd.1.trial098.stderr:2026-02-22T19:50:39.447+0000 7f6ab0fc38c0 -1 Falling back to public interface 2026-02-22T19:50:39.828 INFO:tasks.ceph.cluster2.mgr.x.trial012.stderr:2026-02-22T19:50:39.820+0000 7fc15a9f1640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T19:50:43.210 INFO:tasks.ceph.cluster2.osd.2.trial098.stderr:2026-02-22T19:50:43.207+0000 7f3ec19ac8c0 -1 osd.2 0 log_to_monitors true 2026-02-22T19:50:43.438 INFO:tasks.ceph.cluster2.osd.0.trial098.stderr:2026-02-22T19:50:43.435+0000 7f21abd3b8c0 -1 osd.0 0 log_to_monitors true 2026-02-22T19:50:43.650 INFO:tasks.ceph.cluster2.osd.1.trial098.stderr:2026-02-22T19:50:43.647+0000 7f6ab0fc38c0 -1 osd.1 0 log_to_monitors true 2026-02-22T19:50:44.512 DEBUG:teuthology.orchestra.run.trial098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T19:50:44.658 INFO:teuthology.misc.health.trial098.stdout: 2026-02-22T19:50:44.658 INFO:teuthology.misc.health.trial098.stdout:{"epoch":6,"fsid":"c78b27d3-80f9-4252-ae60-d838c71871f8","created":"2026-02-22T19:50:37.162442+0000","modified":"2026-02-22T19:50:43.827836+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T19:50:37.678649+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"0b31b72f-873e-4231-92d2-2bf473c7fb49","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"12f18cab-1f27-4a1f-9342-68723365376d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"96497b35-d35f-466d-a113-5b0925d9f0c8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:44.667 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T19:50:44.832 INFO:tasks.ceph.cluster2.osd.0.trial098.stderr:2026-02-22T19:50:44.831+0000 7f21a7ce4640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T19:50:44.833 INFO:tasks.ceph.cluster2.osd.1.trial098.stderr:2026-02-22T19:50:44.831+0000 7f6aacf6c640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T19:50:44.833 INFO:tasks.ceph.cluster2.osd.2.trial098.stderr:2026-02-22T19:50:44.831+0000 7f3ebd955640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T19:50:44.852 INFO:tasks.ceph.cluster2.osd.0.trial098.stderr:2026-02-22T19:50:44.851+0000 7f219fb17640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T19:50:44.852 INFO:tasks.ceph.cluster2.osd.1.trial098.stderr:2026-02-22T19:50:44.851+0000 7f6aa4d9f640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T19:50:44.853 INFO:tasks.ceph.cluster2.osd.2.trial098.stderr:2026-02-22T19:50:44.851+0000 7f3eb5788640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T19:50:50.970 DEBUG:teuthology.orchestra.run.trial098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T19:50:51.109 INFO:teuthology.misc.health.trial098.stdout: 2026-02-22T19:50:51.110 INFO:teuthology.misc.health.trial098.stdout:{"epoch":12,"fsid":"c78b27d3-80f9-4252-ae60-d838c71871f8","created":"2026-02-22T19:50:37.162442+0000","modified":"2026-02-22T19:50:49.843584+0000","last_up_change":"2026-02-22T19:50:45.832310+0000","last_in_change":"2026-02-22T19:50:37.678649+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T19:50:46.859457+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0b31b72f-873e-4231-92d2-2bf473c7fb49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6800","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6801","nonce":1925958386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6802","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6804","nonce":1925958386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6810","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6812","nonce":1925958386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6806","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6808","nonce":1925958386}]},"public_addr":"10.20.193.98:6801/1925958386","cluster_addr":"10.20.193.98:6804/1925958386","heartbeat_back_addr":"10.20.193.98:6812/1925958386","heartbeat_front_addr":"10.20.193.98:6808/1925958386","state":["exists","up"]},{"osd":1,"uuid":"12f18cab-1f27-4a1f-9342-68723365376d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6816","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6817","nonce":2796279274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6818","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6819","nonce":2796279274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6822","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6823","nonce":2796279274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6820","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6821","nonce":2796279274}]},"public_addr":"10.20.193.98:6817/2796279274","cluster_addr":"10.20.193.98:6819/2796279274","heartbeat_back_addr":"10.20.193.98:6823/2796279274","heartbeat_front_addr":"10.20.193.98:6821/2796279274","state":["exists","up"]},{"osd":2,"uuid":"96497b35-d35f-466d-a113-5b0925d9f0c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6803","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6805","nonce":2602009039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6807","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6809","nonce":2602009039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6814","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6815","nonce":2602009039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6811","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6813","nonce":2602009039}]},"public_addr":"10.20.193.98:6805/2602009039","cluster_addr":"10.20.193.98:6809/2602009039","heartbeat_back_addr":"10.20.193.98:6815/2602009039","heartbeat_front_addr":"10.20.193.98:6813/2602009039","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.399936+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.672521+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.246703+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:51.119 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T19:50:51.119 INFO:tasks.ceph:Creating RBD pool 2026-02-22T19:50:51.119 DEBUG:teuthology.orchestra.run.trial098:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-22T19:50:51.852 INFO:teuthology.orchestra.run.trial098.stderr:pool 'rbd' created 2026-02-22T19:50:51.864 DEBUG:teuthology.orchestra.run.trial098:> rbd --cluster cluster2 pool init rbd 2026-02-22T19:50:54.866 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-22T19:50:54.867 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-22T19:50:54.867 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-22T19:50:54.868 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-22T19:50:54.868 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T19:50:55.045 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:55.054 INFO:teuthology.orchestra.run.trial098.stdout:{"epoch":5,"flags":0,"active_gid":4099,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6826","nonce":773024755},{"type":"v1","addr":"10.20.193.12:6827","nonce":773024755}]},"active_addr":"10.20.193.12:6827/773024755","active_change":"2026-02-22T19:50:38.822655+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.12:0","nonce":2721686813}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.12:0","nonce":4141106581}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.12:0","nonce":1550454592}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.12:0","nonce":3209243063}]}]} 2026-02-22T19:50:55.137 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-22T19:50:55.137 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-22T19:50:55.137 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T19:50:55.292 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:55.292 INFO:teuthology.orchestra.run.trial098.stdout:{"epoch":16,"fsid":"c78b27d3-80f9-4252-ae60-d838c71871f8","created":"2026-02-22T19:50:37.162442+0000","modified":"2026-02-22T19:50:54.858686+0000","last_up_change":"2026-02-22T19:50:45.832310+0000","last_in_change":"2026-02-22T19:50:37.678649+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T19:50:46.859457+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T19:50:51.265384+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0b31b72f-873e-4231-92d2-2bf473c7fb49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6800","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6801","nonce":1925958386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6802","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6804","nonce":1925958386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6810","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6812","nonce":1925958386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6806","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6808","nonce":1925958386}]},"public_addr":"10.20.193.98:6801/1925958386","cluster_addr":"10.20.193.98:6804/1925958386","heartbeat_back_addr":"10.20.193.98:6812/1925958386","heartbeat_front_addr":"10.20.193.98:6808/1925958386","state":["exists","up"]},{"osd":1,"uuid":"12f18cab-1f27-4a1f-9342-68723365376d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6816","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6817","nonce":2796279274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6818","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6819","nonce":2796279274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6822","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6823","nonce":2796279274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6820","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6821","nonce":2796279274}]},"public_addr":"10.20.193.98:6817/2796279274","cluster_addr":"10.20.193.98:6819/2796279274","heartbeat_back_addr":"10.20.193.98:6823/2796279274","heartbeat_front_addr":"10.20.193.98:6821/2796279274","state":["exists","up"]},{"osd":2,"uuid":"96497b35-d35f-466d-a113-5b0925d9f0c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6803","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6805","nonce":2602009039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6807","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6809","nonce":2602009039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6814","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6815","nonce":2602009039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6811","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6813","nonce":2602009039}]},"public_addr":"10.20.193.98:6805/2602009039","cluster_addr":"10.20.193.98:6809/2602009039","heartbeat_back_addr":"10.20.193.98:6815/2602009039","heartbeat_front_addr":"10.20.193.98:6813/2602009039","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.399936+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.672521+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.246703+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:55.302 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-22T19:50:55.302 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T19:50:55.447 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:55.447 INFO:teuthology.orchestra.run.trial098.stdout:{"epoch":16,"fsid":"c78b27d3-80f9-4252-ae60-d838c71871f8","created":"2026-02-22T19:50:37.162442+0000","modified":"2026-02-22T19:50:54.858686+0000","last_up_change":"2026-02-22T19:50:45.832310+0000","last_in_change":"2026-02-22T19:50:37.678649+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T19:50:46.859457+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T19:50:51.265384+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0b31b72f-873e-4231-92d2-2bf473c7fb49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6800","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6801","nonce":1925958386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6802","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6804","nonce":1925958386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6810","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6812","nonce":1925958386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6806","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6808","nonce":1925958386}]},"public_addr":"10.20.193.98:6801/1925958386","cluster_addr":"10.20.193.98:6804/1925958386","heartbeat_back_addr":"10.20.193.98:6812/1925958386","heartbeat_front_addr":"10.20.193.98:6808/1925958386","state":["exists","up"]},{"osd":1,"uuid":"12f18cab-1f27-4a1f-9342-68723365376d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6816","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6817","nonce":2796279274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6818","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6819","nonce":2796279274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6822","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6823","nonce":2796279274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6820","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6821","nonce":2796279274}]},"public_addr":"10.20.193.98:6817/2796279274","cluster_addr":"10.20.193.98:6819/2796279274","heartbeat_back_addr":"10.20.193.98:6823/2796279274","heartbeat_front_addr":"10.20.193.98:6821/2796279274","state":["exists","up"]},{"osd":2,"uuid":"96497b35-d35f-466d-a113-5b0925d9f0c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6803","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6805","nonce":2602009039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6807","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6809","nonce":2602009039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6814","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6815","nonce":2602009039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6811","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6813","nonce":2602009039}]},"public_addr":"10.20.193.98:6805/2602009039","cluster_addr":"10.20.193.98:6809/2602009039","heartbeat_back_addr":"10.20.193.98:6815/2602009039","heartbeat_front_addr":"10.20.193.98:6813/2602009039","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.399936+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.672521+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.246703+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T19:50:55.458 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-22T19:50:55.459 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-22T19:50:55.459 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-22T19:50:55.541 INFO:teuthology.orchestra.run.trial098.stdout:34359738372 2026-02-22T19:50:55.541 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T19:50:55.543 INFO:teuthology.orchestra.run.trial098.stdout:34359738372 2026-02-22T19:50:55.544 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T19:50:55.545 INFO:teuthology.orchestra.run.trial098.stdout:34359738373 2026-02-22T19:50:55.546 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T19:50:55.688 INFO:teuthology.orchestra.run.trial098.stdout:34359738371 2026-02-22T19:50:55.701 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-22T19:50:55.732 INFO:teuthology.orchestra.run.trial098.stdout:34359738371 2026-02-22T19:50:55.744 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.2 2026-02-22T19:50:55.746 INFO:teuthology.orchestra.run.trial098.stdout:34359738372 2026-02-22T19:50:55.758 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738372 for osd.1 2026-02-22T19:50:56.702 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T19:50:56.745 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T19:50:56.758 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T19:50:56.856 INFO:teuthology.orchestra.run.trial098.stdout:34359738371 2026-02-22T19:50:56.868 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-22T19:50:56.891 INFO:teuthology.orchestra.run.trial098.stdout:34359738372 2026-02-22T19:50:56.903 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.2 2026-02-22T19:50:56.903 DEBUG:teuthology.parallel:result is None 2026-02-22T19:50:56.956 INFO:teuthology.orchestra.run.trial098.stdout:34359738373 2026-02-22T19:50:56.968 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.1 2026-02-22T19:50:56.969 DEBUG:teuthology.parallel:result is None 2026-02-22T19:50:57.869 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T19:50:58.028 INFO:teuthology.orchestra.run.trial098.stdout:34359738372 2026-02-22T19:50:58.039 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.0 2026-02-22T19:50:58.039 DEBUG:teuthology.parallel:result is None 2026-02-22T19:50:58.039 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T19:50:58.039 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T19:50:58.182 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:58.183 INFO:teuthology.orchestra.run.trial098.stderr:dumped all 2026-02-22T19:50:58.194 INFO:teuthology.orchestra.run.trial098.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T19:50:57.120559+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81444,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283034076,"statfs":{"total":289910292480,"available":289826893824,"internally_reserved":0,"allocated":540672,"data_stored":1303096,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.294394"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861563+0000","last_change":"2026-02-22T19:50:54.861563+0000","last_active":"2026-02-22T19:50:54.861563+0000","last_peered":"2026-02-22T19:50:54.861563+0000","last_clean":"2026-02-22T19:50:54.861563+0000","last_became_active":"2026-02-22T19:50:52.861375+0000","last_became_peered":"2026-02-22T19:50:52.861375+0000","last_unstale":"2026-02-22T19:50:54.861563+0000","last_undegraded":"2026-02-22T19:50:54.861563+0000","last_fullsized":"2026-02-22T19:50:54.861563+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:48.919040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030579899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861720+0000","last_change":"2026-02-22T19:50:54.861720+0000","last_active":"2026-02-22T19:50:54.861720+0000","last_peered":"2026-02-22T19:50:54.861720+0000","last_clean":"2026-02-22T19:50:54.861720+0000","last_became_active":"2026-02-22T19:50:52.861559+0000","last_became_peered":"2026-02-22T19:50:52.861559+0000","last_unstale":"2026-02-22T19:50:54.861720+0000","last_undegraded":"2026-02-22T19:50:54.861720+0000","last_fullsized":"2026-02-22T19:50:54.861720+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:54.548580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042885800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861777+0000","last_change":"2026-02-22T19:50:54.861777+0000","last_active":"2026-02-22T19:50:54.861777+0000","last_peered":"2026-02-22T19:50:54.861777+0000","last_clean":"2026-02-22T19:50:54.861777+0000","last_became_active":"2026-02-22T19:50:52.861577+0000","last_became_peered":"2026-02-22T19:50:52.861577+0000","last_unstale":"2026-02-22T19:50:54.861777+0000","last_undegraded":"2026-02-22T19:50:54.861777+0000","last_fullsized":"2026-02-22T19:50:54.861777+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:41:03.314267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046371299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861943+0000","last_change":"2026-02-22T19:50:54.861943+0000","last_active":"2026-02-22T19:50:54.861943+0000","last_peered":"2026-02-22T19:50:54.861943+0000","last_clean":"2026-02-22T19:50:54.861943+0000","last_became_active":"2026-02-22T19:50:52.860973+0000","last_became_peered":"2026-02-22T19:50:52.860973+0000","last_unstale":"2026-02-22T19:50:54.861943+0000","last_undegraded":"2026-02-22T19:50:54.861943+0000","last_fullsized":"2026-02-22T19:50:54.861943+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:16.079406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063366900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.863714+0000","last_change":"2026-02-22T19:50:54.863357+0000","last_active":"2026-02-22T19:50:54.863714+0000","last_peered":"2026-02-22T19:50:54.863714+0000","last_clean":"2026-02-22T19:50:54.863714+0000","last_became_active":"2026-02-22T19:50:52.861077+0000","last_became_peered":"2026-02-22T19:50:52.861077+0000","last_unstale":"2026-02-22T19:50:54.863714+0000","last_undegraded":"2026-02-22T19:50:54.863714+0000","last_fullsized":"2026-02-22T19:50:54.863714+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:21:08.931023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064508999999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:55.134568+0000","last_change":"2026-02-22T19:50:55.134568+0000","last_active":"2026-02-22T19:50:55.134568+0000","last_peered":"2026-02-22T19:50:55.134568+0000","last_clean":"2026-02-22T19:50:55.134568+0000","last_became_active":"2026-02-22T19:50:52.860722+0000","last_became_peered":"2026-02-22T19:50:52.860722+0000","last_unstale":"2026-02-22T19:50:55.134568+0000","last_undegraded":"2026-02-22T19:50:55.134568+0000","last_fullsized":"2026-02-22T19:50:55.134568+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:06:27.453930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026535399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:55.134562+0000","last_change":"2026-02-22T19:50:55.134562+0000","last_active":"2026-02-22T19:50:55.134562+0000","last_peered":"2026-02-22T19:50:55.134562+0000","last_clean":"2026-02-22T19:50:55.134562+0000","last_became_active":"2026-02-22T19:50:52.860764+0000","last_became_peered":"2026-02-22T19:50:52.860764+0000","last_unstale":"2026-02-22T19:50:55.134562+0000","last_undegraded":"2026-02-22T19:50:55.134562+0000","last_fullsized":"2026-02-22T19:50:55.134562+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:34:00.906689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028186500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861491+0000","last_change":"2026-02-22T19:50:54.861491+0000","last_active":"2026-02-22T19:50:54.861491+0000","last_peered":"2026-02-22T19:50:54.861491+0000","last_clean":"2026-02-22T19:50:54.861491+0000","last_became_active":"2026-02-22T19:50:52.860289+0000","last_became_peered":"2026-02-22T19:50:52.860289+0000","last_unstale":"2026-02-22T19:50:54.861491+0000","last_undegraded":"2026-02-22T19:50:54.861491+0000","last_fullsized":"2026-02-22T19:50:54.861491+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:13:55.531886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027309799999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":109,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861173+0000","last_change":"2026-02-22T19:50:48.847243+0000","last_active":"2026-02-22T19:50:54.861173+0000","last_peered":"2026-02-22T19:50:54.861173+0000","last_clean":"2026-02-22T19:50:54.861173+0000","last_became_active":"2026-02-22T19:50:48.846636+0000","last_became_peered":"2026-02-22T19:50:48.846636+0000","last_unstale":"2026-02-22T19:50:54.861173+0000","last_undegraded":"2026-02-22T19:50:54.861173+0000","last_fullsized":"2026-02-22T19:50:54.861173+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_clean_scrub_stamp":"2026-02-22T19:50:47.840459+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:42:30.969871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27108,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344732,"statfs":{"total":96636764160,"available":96609005568,"internally_reserved":0,"allocated":139264,"data_stored":40774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738374,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":200704,"data_stored":631161,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":200704,"data_stored":631161,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T19:50:58.196 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T19:50:58.336 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:58.336 INFO:teuthology.orchestra.run.trial098.stderr:dumped all 2026-02-22T19:50:58.346 INFO:teuthology.orchestra.run.trial098.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T19:50:57.120559+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81444,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283034076,"statfs":{"total":289910292480,"available":289826893824,"internally_reserved":0,"allocated":540672,"data_stored":1303096,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.294394"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861563+0000","last_change":"2026-02-22T19:50:54.861563+0000","last_active":"2026-02-22T19:50:54.861563+0000","last_peered":"2026-02-22T19:50:54.861563+0000","last_clean":"2026-02-22T19:50:54.861563+0000","last_became_active":"2026-02-22T19:50:52.861375+0000","last_became_peered":"2026-02-22T19:50:52.861375+0000","last_unstale":"2026-02-22T19:50:54.861563+0000","last_undegraded":"2026-02-22T19:50:54.861563+0000","last_fullsized":"2026-02-22T19:50:54.861563+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:48.919040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030579899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861720+0000","last_change":"2026-02-22T19:50:54.861720+0000","last_active":"2026-02-22T19:50:54.861720+0000","last_peered":"2026-02-22T19:50:54.861720+0000","last_clean":"2026-02-22T19:50:54.861720+0000","last_became_active":"2026-02-22T19:50:52.861559+0000","last_became_peered":"2026-02-22T19:50:52.861559+0000","last_unstale":"2026-02-22T19:50:54.861720+0000","last_undegraded":"2026-02-22T19:50:54.861720+0000","last_fullsized":"2026-02-22T19:50:54.861720+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:54.548580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042885800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861777+0000","last_change":"2026-02-22T19:50:54.861777+0000","last_active":"2026-02-22T19:50:54.861777+0000","last_peered":"2026-02-22T19:50:54.861777+0000","last_clean":"2026-02-22T19:50:54.861777+0000","last_became_active":"2026-02-22T19:50:52.861577+0000","last_became_peered":"2026-02-22T19:50:52.861577+0000","last_unstale":"2026-02-22T19:50:54.861777+0000","last_undegraded":"2026-02-22T19:50:54.861777+0000","last_fullsized":"2026-02-22T19:50:54.861777+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:41:03.314267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046371299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861943+0000","last_change":"2026-02-22T19:50:54.861943+0000","last_active":"2026-02-22T19:50:54.861943+0000","last_peered":"2026-02-22T19:50:54.861943+0000","last_clean":"2026-02-22T19:50:54.861943+0000","last_became_active":"2026-02-22T19:50:52.860973+0000","last_became_peered":"2026-02-22T19:50:52.860973+0000","last_unstale":"2026-02-22T19:50:54.861943+0000","last_undegraded":"2026-02-22T19:50:54.861943+0000","last_fullsized":"2026-02-22T19:50:54.861943+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:16.079406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063366900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.863714+0000","last_change":"2026-02-22T19:50:54.863357+0000","last_active":"2026-02-22T19:50:54.863714+0000","last_peered":"2026-02-22T19:50:54.863714+0000","last_clean":"2026-02-22T19:50:54.863714+0000","last_became_active":"2026-02-22T19:50:52.861077+0000","last_became_peered":"2026-02-22T19:50:52.861077+0000","last_unstale":"2026-02-22T19:50:54.863714+0000","last_undegraded":"2026-02-22T19:50:54.863714+0000","last_fullsized":"2026-02-22T19:50:54.863714+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:21:08.931023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064508999999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:55.134568+0000","last_change":"2026-02-22T19:50:55.134568+0000","last_active":"2026-02-22T19:50:55.134568+0000","last_peered":"2026-02-22T19:50:55.134568+0000","last_clean":"2026-02-22T19:50:55.134568+0000","last_became_active":"2026-02-22T19:50:52.860722+0000","last_became_peered":"2026-02-22T19:50:52.860722+0000","last_unstale":"2026-02-22T19:50:55.134568+0000","last_undegraded":"2026-02-22T19:50:55.134568+0000","last_fullsized":"2026-02-22T19:50:55.134568+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:06:27.453930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026535399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:55.134562+0000","last_change":"2026-02-22T19:50:55.134562+0000","last_active":"2026-02-22T19:50:55.134562+0000","last_peered":"2026-02-22T19:50:55.134562+0000","last_clean":"2026-02-22T19:50:55.134562+0000","last_became_active":"2026-02-22T19:50:52.860764+0000","last_became_peered":"2026-02-22T19:50:52.860764+0000","last_unstale":"2026-02-22T19:50:55.134562+0000","last_undegraded":"2026-02-22T19:50:55.134562+0000","last_fullsized":"2026-02-22T19:50:55.134562+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:34:00.906689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028186500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861491+0000","last_change":"2026-02-22T19:50:54.861491+0000","last_active":"2026-02-22T19:50:54.861491+0000","last_peered":"2026-02-22T19:50:54.861491+0000","last_clean":"2026-02-22T19:50:54.861491+0000","last_became_active":"2026-02-22T19:50:52.860289+0000","last_became_peered":"2026-02-22T19:50:52.860289+0000","last_unstale":"2026-02-22T19:50:54.861491+0000","last_undegraded":"2026-02-22T19:50:54.861491+0000","last_fullsized":"2026-02-22T19:50:54.861491+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:13:55.531886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027309799999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":109,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T19:50:54.861173+0000","last_change":"2026-02-22T19:50:48.847243+0000","last_active":"2026-02-22T19:50:54.861173+0000","last_peered":"2026-02-22T19:50:54.861173+0000","last_clean":"2026-02-22T19:50:54.861173+0000","last_became_active":"2026-02-22T19:50:48.846636+0000","last_became_peered":"2026-02-22T19:50:48.846636+0000","last_unstale":"2026-02-22T19:50:54.861173+0000","last_undegraded":"2026-02-22T19:50:54.861173+0000","last_fullsized":"2026-02-22T19:50:54.861173+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_clean_scrub_stamp":"2026-02-22T19:50:47.840459+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:42:30.969871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27108,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344732,"statfs":{"total":96636764160,"available":96609005568,"internally_reserved":0,"allocated":139264,"data_stored":40774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738374,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":200704,"data_stored":631161,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":200704,"data_stored":631161,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T19:50:58.347 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T19:50:58.347 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-22T19:50:58.347 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-22T19:50:58.347 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-22T19:50:58.508 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T19:50:58.509 INFO:teuthology.orchestra.run.trial098.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T19:50:58.521 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-22T19:50:58.522 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T19:50:58.538 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T19:50:58.538 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial098.front.sepia.ceph.com 2026-02-22T19:50:58.538 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:50:58.717 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.4] 2026-02-22T19:50:58.717 INFO:teuthology.orchestra.run.trial098.stdout: key = AQDbXZtpUBlWMhAA3+lf2FUjsi1BBVP5wbadTg== 2026-02-22T19:50:58.717 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:50:58.717 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:50:58.717 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.4 2026-02-22T19:50:58.728 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:50:58.909 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.5] 2026-02-22T19:50:58.909 INFO:teuthology.orchestra.run.trial098.stdout: key = AQDbXZtp5XMoNhAAj9W31gTaF5YARYUznYCj+w== 2026-02-22T19:50:58.910 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:50:58.910 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:50:58.910 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.5 2026-02-22T19:50:58.920 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:50:59.101 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.6] 2026-02-22T19:50:59.102 INFO:teuthology.orchestra.run.trial098.stdout: key = AQDbXZtpCm/5ORAAPbmb6hO+u30HxakLvq69Tw== 2026-02-22T19:50:59.102 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:50:59.102 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:50:59.102 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.6 2026-02-22T19:50:59.112 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:50:59.297 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.4] 2026-02-22T19:50:59.298 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpGKS7DRAAlT/BeRWIyejFbzhK6SHEKg== 2026-02-22T19:50:59.298 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:50:59.298 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:50:59.298 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.4 2026-02-22T19:50:59.309 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:50:59.488 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.5] 2026-02-22T19:50:59.489 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpPJgKEhAAIiCHZhxJhwiIB0uwwNo/ag== 2026-02-22T19:50:59.489 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:50:59.489 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:50:59.489 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.5 2026-02-22T19:50:59.503 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:50:59.679 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.6] 2026-02-22T19:50:59.679 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpIahWFhAAZke4pzDPATr7dQdTV5Eqnw== 2026-02-22T19:50:59.679 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:50:59.679 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:50:59.679 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.6 2026-02-22T19:50:59.690 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T19:50:59.704 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T19:50:59.705 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial098.front.sepia.ceph.com 2026-02-22T19:50:59.705 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:50:59.888 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror] 2026-02-22T19:50:59.888 INFO:teuthology.orchestra.run.trial098.stdout: key = AQDbXZtpcC64HRAA540s/XyzUc0NJrT55rLP+Q== 2026-02-22T19:50:59.888 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T19:50:59.888 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:50:59.889 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror 2026-02-22T19:50:59.899 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:51:00.076 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.0] 2026-02-22T19:51:00.076 INFO:teuthology.orchestra.run.trial098.stdout: key = AQDbXZtpFse2IRAA/8HhEZINBaoNlABmgrGWcg== 2026-02-22T19:51:00.076 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:51:00.076 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:51:00.076 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.0 2026-02-22T19:51:00.087 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:51:00.268 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.1] 2026-02-22T19:51:00.268 INFO:teuthology.orchestra.run.trial098.stdout: key = AQDbXZtpDj7WJRAA7dG2JSHoKBqmdIKDgthDMg== 2026-02-22T19:51:00.268 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:51:00.269 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:51:00.269 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.1 2026-02-22T19:51:00.279 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:51:00.464 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.2] 2026-02-22T19:51:00.465 INFO:teuthology.orchestra.run.trial098.stdout: key = AQDbXZtpQygaKhAAKjzPZCa0M8bH6Y40zAJz0g== 2026-02-22T19:51:00.465 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:51:00.465 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:51:00.465 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.2 2026-02-22T19:51:00.478 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:51:00.660 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.3] 2026-02-22T19:51:00.660 INFO:teuthology.orchestra.run.trial098.stdout: key = AQDbXZtp2IpmLhAAmkI1Mkj+iXmUJVesqc7JKg== 2026-02-22T19:51:00.660 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:51:00.661 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:51:00.661 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.3 2026-02-22T19:51:00.673 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:51:00.851 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror] 2026-02-22T19:51:00.851 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD+XZtprn9PNRAA2CzkRwyKAn5PTTq28VFsOw== 2026-02-22T19:51:00.851 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T19:51:00.851 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:51:00.852 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror 2026-02-22T19:51:00.862 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:51:01.042 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.0] 2026-02-22T19:51:01.042 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD+XZtpVsBPORAAKyxXRz4sKTMFqOx0koobxA== 2026-02-22T19:51:01.042 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:51:01.042 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:51:01.043 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.0 2026-02-22T19:51:01.053 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:51:01.228 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.1] 2026-02-22T19:51:01.228 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpbmTMARAA0la633W88D+eYp9UOAw5Qg== 2026-02-22T19:51:01.229 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:51:01.229 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:51:01.229 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.1 2026-02-22T19:51:01.239 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:51:01.421 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.2] 2026-02-22T19:51:01.421 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtpZyPBBRAACdv2YW5tTz1dmTF/8Nxqhw== 2026-02-22T19:51:01.421 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:51:01.421 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:51:01.422 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.2 2026-02-22T19:51:01.433 DEBUG:teuthology.orchestra.run.trial098:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T19:51:01.609 INFO:teuthology.orchestra.run.trial098.stdout:[client.mirror.3] 2026-02-22T19:51:01.609 INFO:teuthology.orchestra.run.trial098.stdout: key = AQD/XZtp++WVCRAAhcsl0QepXe2Q/mHbJKs+Og== 2026-02-22T19:51:01.610 INFO:teuthology.orchestra.run.trial098.stdout: caps mon = "profile rbd-mirror" 2026-02-22T19:51:01.610 INFO:teuthology.orchestra.run.trial098.stdout: caps osd = "profile rbd" 2026-02-22T19:51:01.610 INFO:teuthology.orchestra.run.trial098.stderr:updated caps for client.mirror.3 2026-02-22T19:51:01.620 INFO:teuthology.run_tasks:Running task workunit... 2026-02-22T19:51:01.637 INFO:tasks.workunit:Pulling workunits from ref f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T19:51:01.637 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-22T19:51:01.637 DEBUG:teuthology.orchestra.run.trial098:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T19:51:01.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T19:51:01.641 INFO:teuthology.orchestra.run.trial098.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-22T19:51:01.641 DEBUG:teuthology.orchestra.run.trial098:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T19:51:01.686 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T19:51:01.686 DEBUG:teuthology.orchestra.run.trial098:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-22T19:51:01.730 INFO:tasks.workunit:timeout=3h 2026-02-22T19:51:01.730 INFO:tasks.workunit:cleanup=True 2026-02-22T19:51:01.730 DEBUG:teuthology.orchestra.run.trial098:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T19:51:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-22T19:51:23.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Note: switching to 'f90f4eaebb28b619cf76e80cbbe997656da69579'. 2026-02-22T19:51:23.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T19:51:23.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-22T19:51:23.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:state without impacting any branches by switching back to a branch. 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: git switch -c 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Or undo this operation with: 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: git switch - 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-22T19:51:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T19:51:23.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:HEAD is now at f90f4eaebb2 Merge branch 'wip-74712-squid' of https://github.com/idryomov/ceph into wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T19:51:23.743 DEBUG:teuthology.orchestra.run.trial098:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-22T19:51:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-22T19:51:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T19:51:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-22T19:51:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-22T19:51:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-22T19:51:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T19:51:23.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T19:51:23.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-22T19:51:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T19:51:23.972 DEBUG:teuthology.orchestra.run.trial098:> set -ex 2026-02-22T19:51:23.973 DEBUG:teuthology.orchestra.run.trial098:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-22T19:51:24.019 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror.sh on cluster1.client.mirror... 2026-02-22T19:51:24.022 INFO:tasks.workunit:Running workunit rbd/rbd_mirror.sh... 2026-02-22T19:51:24.022 DEBUG:teuthology.orchestra.run.trial098:workunit test rbd/rbd_mirror.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f90f4eaebb28b619cf76e80cbbe997656da69579 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' RBD_IMAGE_FEATURES=layering RBD_MIRROR_CONFIG_KEY=1 RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-22T19:51:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -gt 0 ']' 2026-02-22T19:51:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot '!=' snapshot ']' 2026-02-22T19:51:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ type xmlstarlet 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ XMLSTARLET=xmlstarlet 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CLUSTER1=cluster1 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CLUSTER2=cluster2 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ POOL=mirror 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ PARENT_POOL=mirror_parent 2026-02-22T19:51:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ NS1=ns1 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ NS2=ns2 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ TEMPDIR= 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ID=mirror 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ RBD_IMAGE_FEATURES=layering 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ RBD_MIRROR_MODE=snapshot 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ MIRROR_POOL_MODE=pool 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' snapshot = snapshot ']' 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ MIRROR_POOL_MODE=image 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS='--id mirror' 2026-02-22T19:51:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-22T19:51:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-22T19:51:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-22T19:51:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-22T19:51:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_BIN=. 2026-02-22T19:51:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_SRC=. 2026-02-22T19:51:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-22T19:51:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 -gt 0 ']' 2026-02-22T19:51:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ setup 2026-02-22T19:51:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local c 2026-02-22T19:51:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-22T19:51:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ setup_tempdir 2026-02-22T19:51:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mktemp -d 2026-02-22T19:51:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ TEMPDIR=/tmp/tmp.5OeMBFOFTF 2026-02-22T19:51:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z 1 ']' 2026-02-22T19:51:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ setup_pools cluster1 cluster2 2026-02-22T19:51:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local remote_cluster=cluster2 2026-02-22T19:51:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mon_map_file 2026-02-22T19:51:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mon_addr 2026-02-22T19:51:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local admin_key_file 2026-02-22T19:51:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local uuid 2026-02-22T19:51:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-22T19:51:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:pool 'mirror' already exists 2026-02-22T19:51:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-22T19:51:26.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:pool 'mirror_parent' already exists 2026-02-22T19:51:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-22T19:51:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-22T19:51:31.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 1 ']' 2026-02-22T19:51:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ PEER_CLUSTER_SUFFIX=-DNE 2026-02-22T19:51:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1-DNE mirror image 2026-02-22T19:51:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-22T19:51:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-22T19:51:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-22T19:51:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-22T19:51:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-02-22T19:51:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-22T19:51:32.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 image 2026-02-22T19:51:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z ']' 2026-02-22T19:51:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z 1 ']' 2026-02-22T19:51:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mon_map_file=/tmp/tmp.5OeMBFOFTF/cluster2.monmap 2026-02-22T19:51:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster2 mon getmap 2026-02-22T19:51:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:got monmap epoch 1 2026-02-22T19:51:32.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ monmaptool --print /tmp/tmp.5OeMBFOFTF/cluster2.monmap 2026-02-22T19:51:32.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -E 'mon\.' 2026-02-22T19:51:32.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ head -n 1 2026-02-22T19:51:32.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' 2026-02-22T19:51:32.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -E 's/\/[0-9]+//g' 2026-02-22T19:51:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mon_addr='[v2:10.20.193.98:3300,v1:10.20.193.98:6789]' 2026-02-22T19:51:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_key_file=/tmp/tmp.5OeMBFOFTF/cluster2.client.mirror.key 2026-02-22T19:51:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster2 auth get-key client.mirror 2026-02-22T19:51:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ peer_add cluster1 mirror client.mirror@cluster2-DNE '' --remote-mon-host '[v2:10.20.193.98:3300,v1:10.20.193.98:6789]' --remote-key-file /tmp/tmp.5OeMBFOFTF/cluster2.client.mirror.key 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local client_cluster=client.mirror@cluster2-DNE 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local remote_cluster=cluster2-DNE 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local uuid_var_name 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local error_code 2026-02-22T19:51:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local peer_uuid 2026-02-22T19:51:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T19:51:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:51:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror client.mirror@cluster2-DNE --remote-mon-host '[v2:10.20.193.98:3300,v1:10.20.193.98:6789]' --remote-key-file /tmp/tmp.5OeMBFOFTF/cluster2.client.mirror.key 2026-02-22T19:51:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ peer_uuid=df7543bc-b569-4771-a1ad-0eef6af1c1d2 2026-02-22T19:51:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 -eq 0 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ peer_add cluster1 mirror_parent client.mirror@cluster2-DNE uuid 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local client_cluster=client.mirror@cluster2-DNE 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local remote_cluster=cluster2-DNE 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local uuid_var_name 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n uuid ']' 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ uuid_var_name=uuid 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local error_code 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local peer_uuid 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:51:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent client.mirror@cluster2-DNE 2026-02-22T19:51:33.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ peer_uuid=c1b1b313-242d-44ac-bb04-cf62de357d26 2026-02-22T19:51:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:51:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:51:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T19:51:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 -eq 0 2026-02-22T19:51:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n uuid ']' 2026-02-22T19:51:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval uuid=c1b1b313-242d-44ac-bb04-cf62de357d26 2026-02-22T19:51:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ uuid=c1b1b313-242d-44ac-bb04-cf62de357d26 2026-02-22T19:51:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror pool peer set mirror_parent c1b1b313-242d-44ac-bb04-cf62de357d26 mon-host '[v2:10.20.193.98:3300,v1:10.20.193.98:6789]' 2026-02-22T19:51:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror pool peer set mirror_parent c1b1b313-242d-44ac-bb04-cf62de357d26 key-file /tmp/tmp.5OeMBFOFTF/cluster2.client.mirror.key 2026-02-22T19:51:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ setup_pools cluster2 cluster1 2026-02-22T19:51:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local remote_cluster=cluster1 2026-02-22T19:51:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mon_map_file 2026-02-22T19:51:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mon_addr 2026-02-22T19:51:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local admin_key_file 2026-02-22T19:51:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local uuid 2026-02-22T19:51:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:33.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-22T19:51:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:pool 'mirror' already exists 2026-02-22T19:51:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-22T19:51:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:pool 'mirror_parent' already exists 2026-02-22T19:51:34.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:34.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-22T19:51:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-22T19:51:40.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 1 ']' 2026-02-22T19:51:40.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ PEER_CLUSTER_SUFFIX=-DNE 2026-02-22T19:51:40.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:40.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2-DNE mirror image 2026-02-22T19:51:40.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-22T19:51:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-22T19:51:40.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-22T19:51:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-22T19:51:41.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-02-22T19:51:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-22T19:51:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 image 2026-02-22T19:51:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z ']' 2026-02-22T19:51:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z 1 ']' 2026-02-22T19:51:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mon_map_file=/tmp/tmp.5OeMBFOFTF/cluster1.monmap 2026-02-22T19:51:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster1 mon getmap 2026-02-22T19:51:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:got monmap epoch 1 2026-02-22T19:51:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ monmaptool --print /tmp/tmp.5OeMBFOFTF/cluster1.monmap 2026-02-22T19:51:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -E 'mon\.' 2026-02-22T19:51:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ head -n 1 2026-02-22T19:51:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' 2026-02-22T19:51:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -E 's/\/[0-9]+//g' 2026-02-22T19:51:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mon_addr='[v2:10.20.193.12:3300,v1:10.20.193.12:6789]' 2026-02-22T19:51:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_key_file=/tmp/tmp.5OeMBFOFTF/cluster1.client.mirror.key 2026-02-22T19:51:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster1 auth get-key client.mirror 2026-02-22T19:51:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ peer_add cluster2 mirror client.mirror@cluster1-DNE '' --remote-mon-host '[v2:10.20.193.12:3300,v1:10.20.193.12:6789]' --remote-key-file /tmp/tmp.5OeMBFOFTF/cluster1.client.mirror.key 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local client_cluster=client.mirror@cluster1-DNE 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local remote_cluster=cluster1-DNE 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local uuid_var_name 2026-02-22T19:51:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local error_code 2026-02-22T19:51:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local peer_uuid 2026-02-22T19:51:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T19:51:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:51:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror client.mirror@cluster1-DNE --remote-mon-host '[v2:10.20.193.12:3300,v1:10.20.193.12:6789]' --remote-key-file /tmp/tmp.5OeMBFOFTF/cluster1.client.mirror.key 2026-02-22T19:51:41.696 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ peer_uuid=e56cde10-e99e-4426-8926-9ba4aec5da6d 2026-02-22T19:51:41.696 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:51:41.696 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:51:41.696 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T19:51:41.696 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 -eq 0 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ peer_add cluster2 mirror_parent client.mirror@cluster1-DNE uuid 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local client_cluster=client.mirror@cluster1-DNE 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local remote_cluster=cluster1-DNE 2026-02-22T19:51:41.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local uuid_var_name 2026-02-22T19:51:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n uuid ']' 2026-02-22T19:51:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ uuid_var_name=uuid 2026-02-22T19:51:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local error_code 2026-02-22T19:51:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local peer_uuid 2026-02-22T19:51:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T19:51:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:51:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent client.mirror@cluster1-DNE 2026-02-22T19:51:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ peer_uuid=d90d953f-fb15-4d28-ab1a-92a585a39caf 2026-02-22T19:51:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:51:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:51:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T19:51:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 -eq 0 2026-02-22T19:51:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n uuid ']' 2026-02-22T19:51:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval uuid=d90d953f-fb15-4d28-ab1a-92a585a39caf 2026-02-22T19:51:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ uuid=d90d953f-fb15-4d28-ab1a-92a585a39caf 2026-02-22T19:51:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror pool peer set mirror_parent d90d953f-fb15-4d28-ab1a-92a585a39caf mon-host '[v2:10.20.193.12:3300,v1:10.20.193.12:6789]' 2026-02-22T19:51:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T19:51:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror pool peer set mirror_parent d90d953f-fb15-4d28-ab1a-92a585a39caf key-file /tmp/tmp.5OeMBFOFTF/cluster1.client.mirror.key 2026-02-22T19:51:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: add image and test replay' 2026-02-22T19:51:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T19:51:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T19:51:41.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 19:51:41 TEST: add image and test replay 2026-02-22T19:51:41.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 19:51:41 TEST: add image and test replay 2026-02-22T19:51:41.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster1 2026-02-22T19:51:41.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:41.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:51:41.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:41.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:0 2026-02-22T19:51:41.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:51:41.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:1 2026-02-22T19:51:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:51:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:51:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:51:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:2 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:51:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:51:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:51:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:51:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:41.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:3 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:51:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:51:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:51:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:51:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image=test 2026-02-22T19:51:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-02-22T19:51:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T19:51:41.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:41.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test 2026-02-22T19:51:41.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T19:51:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T19:51:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-22T19:51:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T19:51:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-22T19:51:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T19:51:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-22T19:51:42.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test': 2026-02-22T19:51:42.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T19:51:42.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 10b5b5178dbe 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.10b5b5178dbe 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 19:51:41 2026 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 19:51:41 2026 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 19:51:41 2026 2026-02-22T19:51:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T19:51:42.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T19:51:42.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:51:42.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T19:51:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_image_meta cluster2 mirror test key1 value1 2026-02-22T19:51:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local key=key1 2026-02-22T19:51:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=value1 2026-02-22T19:51:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key1 value1 2026-02-22T19:51:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_image_meta cluster2 mirror test key2 value2 2026-02-22T19:51:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local key=key2 2026-02-22T19:51:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=value2 2026-02-22T19:51:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key2 value2 2026-02-22T19:51:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:51:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:51:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:51:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:51:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:51:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:51:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:51:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:51:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:51:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:51:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:51:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:51:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:51:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:51:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:51:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:51:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:51:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:51:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:51:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:51:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:51:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:51:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:51:44.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:51:44.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:51:44.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:51:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:51:44.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:51:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:51:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:51:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:51:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:51:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:51:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:51:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T19:51:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T19:51:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:51:44.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:44.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:44.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:44.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T19:51:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:51:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:51:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:51:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:51:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:51:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T19:51:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T19:51:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:51:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:44.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:44.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:44.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T19:51:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:51:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:51:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:51:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T19:51:45.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:51:45.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:51:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:51:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:51:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:51:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:51:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:51:45.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:51:45.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:51:45.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:51:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:51:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test 100 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T19:51:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T19:51:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T19:51:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T19:51:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 12500.3 bytes/sec: 49 MiB/s 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:51:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 4 2026-02-22T19:51:45.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:51:45.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:51:45.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:45.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:45.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:45.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:51:45.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:51:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:51:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:51:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:51:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:51:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:51:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:51:45.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:51:45.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:45.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:45.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:45.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:51:45.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:51:45.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:51:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:51:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=4 2026-02-22T19:51:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:51:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:51:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:51:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:51:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:51:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:51:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:51:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:51:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=4 2026-02-22T19:51:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 4 = 4 2026-02-22T19:51:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T19:51:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:51:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:51:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:51:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:51:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:51:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:51:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:51:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:51:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:51:47.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:51:47.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:51:47.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:51:47.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:51:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:51:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:51:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:51:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789902,"replay_state":"idle"} 2026-02-22T19:51:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:51:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:51:44' 2026-02-22T19:51:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:51:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:51:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:51:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789902,"replay_state":"idle"} 2026-02-22T19:51:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:51:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:51:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:51:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:51:47.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:51:47.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:51:47.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:51:44' 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:51:44' 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:51:44' 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:51:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING142694277mirror.0trial09819.2.3-689-gf90f4eaetrueOK42734286mirror.1trial09819.2.3-689-gf90f4eaefalseOK42794289mirror.2trial09819.2.3-689-gf90f4eaefalseOK42854292mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789902,"replay_state":"idle"}42694277mirror.0trial0982026-02-22 19:51:44' 2026-02-22T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:51:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:51:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:51:44' 2026-02-22T19:51:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:51:47.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:51:47.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:51:47.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:51:47.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test down+unknown 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=down+unknown 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:51:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:51:47.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:47.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:51:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test down+unknown '' '' 2026-02-22T19:51:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=down+unknown 2026-02-22T19:51:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:51:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:51:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:51:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:51:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:51:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:51:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:51:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T19:51:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T19:51:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789902,"replay_state":"idle"} 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:51:44 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 3 .mirror.primary.1022b5fb-140c-4e47-a45c-827e5aa56e92.e827414e-ed0f-430f-b706-a579dc50ca13 (peer_uuids:[]) 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 4 .mirror.primary.1022b5fb-140c-4e47-a45c-827e5aa56e92.dfba129f-c09d-46b4-b372-7cc03a2c6ce7 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: down+unknown 2026-02-22T19:51:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:51:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: status not found 2026-02-22T19:51:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo down+unknown 2026-02-22T19:51:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:51:48.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: ' 2026-02-22T19:51:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:51:48.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update= 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test down+unknown '' 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=down+unknown 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local prev_last_update= 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:51:48.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:51:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='UNKNOWNUNKNOWNOK1test1022b5fb-140c-4e47-a45c-827e5aa56e92down+unknownstatus not foundcluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789902,"replay_state":"idle"}2026-02-22 19:51:44' 2026-02-22T19:51:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:51:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:51:48.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update= 2026-02-22T19:51:48.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:51:48.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=down+unknown 2026-02-22T19:51:48.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo down+unknown 2026-02-22T19:51:48.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep down+unknown 2026-02-22T19:51:48.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:down+unknown 2026-02-22T19:51:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test 2026-02-22T19:51:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T19:51:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T19:51:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.export 2026-02-22T19:51:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:51:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:51:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T19:51:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.export 2026-02-22T19:51:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:51:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:51:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:51:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:51:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:51:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:51:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:51:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:51:48.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:51:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_image_meta cluster1 mirror test key1 value1 2026-02-22T19:51:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:48.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:48.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local key=key1 2026-02-22T19:51:48.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local value=value1 2026-02-22T19:51:48.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key1 2026-02-22T19:51:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test value1 = value1 2026-02-22T19:51:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_image_meta cluster1 mirror test key2 value2 2026-02-22T19:51:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:51:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local key=key2 2026-02-22T19:51:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local value=value2 2026-02-22T19:51:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key2 2026-02-22T19:51:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test value2 = value2 2026-02-22T19:51:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: stop mirror, add image, start mirror and test replay' 2026-02-22T19:51:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T19:51:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T19:51:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 19:51:48 TEST: stop mirror, add image, start mirror and test replay 2026-02-22T19:51:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 19:51:48 TEST: stop mirror, add image, start mirror and test replay 2026-02-22T19:51:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster1 2026-02-22T19:51:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T19:51:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:51:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T19:51:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:51:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T19:51:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:0 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T19:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T19:51:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:51:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:51:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T19:51:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T19:51:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T19:51:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=20680 2026-02-22T19:51:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 20680 ']' 2026-02-22T19:51:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 20680 2026-02-22T19:51:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:51:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:51:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20680 '$2 == pid {print; exit 1}' 2026-02-22T19:51:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 20680 1.7 0.0 766940 40248 ? Ssl 19:51 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:51:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:51:49.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:49.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20680 '$2 == pid {print; exit 1}' 2026-02-22T19:51:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T19:51:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:49.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20680 '$2 == pid {print; exit 1}' 2026-02-22T19:51:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:51:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:51:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:51:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:51:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:51:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:49.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:51:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:0 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:51:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T19:51:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T19:51:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T19:51:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T19:51:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:51:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T19:51:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T19:51:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T19:51:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:1 2026-02-22T19:51:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:51:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:51:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T19:51:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:51:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T19:51:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T19:51:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:51:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:51:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T19:51:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T19:51:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T19:51:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=20708 2026-02-22T19:51:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 20708 ']' 2026-02-22T19:51:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 20708 2026-02-22T19:51:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:51:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:51:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20708 '$2 == pid {print; exit 1}' 2026-02-22T19:51:49.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 20708 0.6 0.0 496468 36636 ? Ssl 19:51 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:51:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:51:50.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:50.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20708 '$2 == pid {print; exit 1}' 2026-02-22T19:51:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T19:51:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20708 '$2 == pid {print; exit 1}' 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:51:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:51:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:51:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:51:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:51:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:51:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:51:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:51:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:51:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:1 2026-02-22T19:51:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:51:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:51:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:51:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:51:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:51:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:51:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T19:51:50.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T19:51:50.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T19:51:50.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:50.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T19:51:50.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:51:50.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T19:51:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T19:51:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T19:51:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:2 2026-02-22T19:51:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T19:51:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T19:51:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T19:51:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T19:51:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=20743 2026-02-22T19:51:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 20743 ']' 2026-02-22T19:51:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 20743 2026-02-22T19:51:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:51:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:51:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20743 '$2 == pid {print; exit 1}' 2026-02-22T19:51:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 20743 0.6 0.0 701368 35964 ? Ssl 19:51 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:51:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:51:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:52.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20743 '$2 == pid {print; exit 1}' 2026-02-22T19:51:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T19:51:52.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:52.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20743 '$2 == pid {print; exit 1}' 2026-02-22T19:51:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:51:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:51:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:51:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:51:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:51:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:51:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:51:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:51:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:51:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:51:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:51:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:51:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T19:51:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:51:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:51:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:51:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:51:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:51:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:51:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:2 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:51:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:51:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:51:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T19:51:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T19:51:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T19:51:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T19:51:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:51:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T19:51:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T19:51:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T19:51:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:3 2026-02-22T19:51:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:51:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T19:51:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T19:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T19:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T19:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T19:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T19:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T19:51:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T19:51:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T19:51:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=20796 2026-02-22T19:51:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 20796 ']' 2026-02-22T19:51:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 20796 2026-02-22T19:51:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:51:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:51:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20796 '$2 == pid {print; exit 1}' 2026-02-22T19:51:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 20796 0.7 0.0 701372 35532 ? Ssl 19:51 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:51:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:51:53.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:53.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20796 '$2 == pid {print; exit 1}' 2026-02-22T19:51:53.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T19:51:53.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T19:51:53.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=20796 '$2 == pid {print; exit 1}' 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:51:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:51:53.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:51:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:51:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:51:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:3 2026-02-22T19:51:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T19:51:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:51:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:51:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:51:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:51:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:51:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T19:51:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T19:51:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image1=test1 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test1 snapshot 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T19:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test1 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test1 2026-02-22T19:51:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T19:51:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T19:51:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test1 snapshot 2026-02-22T19:51:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T19:51:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test1 snapshot 2026-02-22T19:51:54.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T19:51:54.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test1 2026-02-22T19:51:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test1': 2026-02-22T19:51:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T19:51:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 10de5b4907d5 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.10de5b4907d5 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 19:51:53 2026 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 19:51:53 2026 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 19:51:53 2026 2026-02-22T19:51:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T19:51:54.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T19:51:54.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:51:54.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T19:51:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T19:51:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T19:51:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T19:51:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:54.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T19:51:54.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T19:51:54.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T19:51:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T19:51:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 12500.3 bytes/sec: 49 MiB/s 2026-02-22T19:51:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster1 2026-02-22T19:51:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:51:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:0 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:51:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:51:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:51:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:1 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:51:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:51:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:51:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:2 2026-02-22T19:51:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:51:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:51:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:51:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:51:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:51:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:51:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:51:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:51:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:51:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:51:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:51:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:54.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:3 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:51:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:51:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:51:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:51:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T19:51:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:51:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T19:51:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:51:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:51:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:51:55.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:51:55.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:51:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:51:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:51:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:51:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:51:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:51:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:51:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:51:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:51:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:51:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:51:55.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T19:51:55.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:51:55.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:51:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:55.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:51:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:51:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:51:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:51:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:51:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:55.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:51:55.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T19:51:55.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T19:51:55.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:51:55.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:51:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T19:51:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T19:51:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:51:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T19:51:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T19:51:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:51:55.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:55.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:55.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:51:55.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T19:51:55.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T19:51:55.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:51:55.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:51:55.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test1 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:51:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T19:51:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:51:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:51:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test1 2026-02-22T19:51:55.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T19:51:55.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:51:55.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:51:55.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:55.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T19:51:56.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:51:56.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:51:56.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:51:56.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:51:56.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:51:56.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:51:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:51:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:51:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:51:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:51:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:51:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:51:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:51:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:51:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:51:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:51:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:51:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:51:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:51:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:51:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test1 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test1 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test1 2026-02-22T19:51:57.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 6 2026-02-22T19:51:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test1 2026-02-22T19:51:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:51:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test1.status 2026-02-22T19:51:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test1.status 2026-02-22T19:51:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:51:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test1.status 2026-02-22T19:51:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test1.status 2026-02-22T19:51:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test1.status 2026-02-22T19:51:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:51:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.status 2026-02-22T19:51:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test1 /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test1.status 2026-02-22T19:51:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:51:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test1.status 2026-02-22T19:51:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test1 --format xml 2026-02-22T19:51:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:51:57.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:51:57.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=6 2026-02-22T19:51:57.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:51:57.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:51:57.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:51:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.status 2026-02-22T19:51:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.status 2026-02-22T19:51:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-02-22T19:51:57.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:51:57.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:51:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=5 2026-02-22T19:51:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 6 = 5 2026-02-22T19:51:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:51:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T19:51:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.status 2026-02-22T19:51:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.status 2026-02-22T19:51:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-02-22T19:51:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:51:57.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:51:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=6 2026-02-22T19:51:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 6 = 6 2026-02-22T19:51:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:51:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test1 2026-02-22T19:51:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:51:57.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:51:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:51:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T19:51:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:51:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:51:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:51:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:51:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:51:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:51:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:51:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:51:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:51:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:51:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:51:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:51:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789914,"replay_state":"idle"} 2026-02-22T19:51:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:51:56' 2026-02-22T19:51:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:51:58.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:51:58.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:51:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789914,"replay_state":"idle"} 2026-02-22T19:51:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:51:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:51:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:51:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:51:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:51:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T19:51:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:51:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:52:00.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T19:52:00.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:00.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:00.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:00.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:00.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:00.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:00.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:00.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:00.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:00.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789914,"replay_state":"idle"} 2026-02-22T19:52:00.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:00.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:51:56' 2026-02-22T19:52:00.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:00.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789914,"replay_state":"idle"} 2026-02-22T19:52:00.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:00.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:00.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:00.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:00.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:51:56' 2026-02-22T19:52:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 19:51:56' 2026-02-22T19:52:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:51:56' 2026-02-22T19:52:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:00.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789905,"remote_snapshot_timestamp":1771789905,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:51:58test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789914,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:51:56' 2026-02-22T19:52:00.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:00.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:51:56' 2026-02-22T19:52:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T19:52:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T19:52:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=down+unknown 2026-02-22T19:52:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:01.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:01.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:01.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T19:52:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=down+unknown 2026-02-22T19:52:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T19:52:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T19:52:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T19:52:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22T19:52:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:52:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:52:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789914,"replay_state":"idle"} 2026-02-22T19:52:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:51:56 2026-02-22T19:52:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:52:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 5 .mirror.primary.ef0d415f-0391-42f8-b805-2bf223ba80c9.aa117db2-d4cc-49ca-b631-590dd12318a1 (peer_uuids:[]) 2026-02-22T19:52:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 6 .mirror.primary.ef0d415f-0391-42f8-b805-2bf223ba80c9.29bf08ca-4244-4ab6-8715-ab99e32d23dd (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:52:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T19:52:02.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: down+unknown 2026-02-22T19:52:02.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: status not found 2026-02-22T19:52:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo down+unknown 2026-02-22T19:52:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: ' 2026-02-22T19:52:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:02.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update= 2026-02-22T19:52:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T19:52:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=down+unknown 2026-02-22T19:52:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local prev_last_update= 2026-02-22T19:52:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:02.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='UNKNOWNUNKNOWNOK2test1022b5fb-140c-4e47-a45c-827e5aa56e92down+unknownstatus not foundcluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789905,"remote_snapshot_timestamp":1771789905,"replay_state":"idle"}2026-02-22 19:51:58test1ef0d415f-0391-42f8-b805-2bf223ba80c9down+unknownstatus not foundcluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789914,"replay_state":"idle"}2026-02-22 19:51:56' 2026-02-22T19:52:02.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:02.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update= 2026-02-22T19:52:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=down+unknown 2026-02-22T19:52:02.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo down+unknown 2026-02-22T19:52:02.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep down+unknown 2026-02-22T19:52:02.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:down+unknown 2026-02-22T19:52:02.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test1 2026-02-22T19:52:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T19:52:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T19:52:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T19:52:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:52:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test1.export 2026-02-22T19:52:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T19:52:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T19:52:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:52:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.export 2026-02-22T19:52:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test1.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.export 2026-02-22T19:52:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test1.export 2026-02-22T19:52:02.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:52:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.export 2026-02-22T19:52:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:52:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test1.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.export 2026-02-22T19:52:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test1.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test1.export 2026-02-22T19:52:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: test the first image is replaying after restart' 2026-02-22T19:52:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T19:52:02.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T19:52:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 19:52:02 TEST: test the first image is replaying after restart 2026-02-22T19:52:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 19:52:02 TEST: test the first image is replaying after restart 2026-02-22T19:52:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test 100 2026-02-22T19:52:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T19:52:02.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T19:52:02.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:52:02.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T19:52:02.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T19:52:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T19:52:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T19:52:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.5 bytes/sec: 33 MiB/s 2026-02-22T19:52:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:52:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:52:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:03.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:03.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:03.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:03.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:03.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:03.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:03.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:52:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:52:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:52:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:52:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:52:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:52:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 7 2026-02-22T19:52:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:52:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:52:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:52:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:52:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:52:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:52:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:52:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:52:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:52:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:52:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:52:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:52:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:52:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:52:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:52:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=7 2026-02-22T19:52:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:52:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:52:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:52:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:52:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:52:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:52:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:52:05.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=7 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 7 = 7 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:52:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:05.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:05.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:05.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:05.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:05.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:05.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:05.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:06.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:52:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789905,"remote_snapshot_timestamp":1771789905,"replay_state":"idle"} 2026-02-22T19:52:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:06.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:51:58' 2026-02-22T19:52:06.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:06.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:06.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:06.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789905,"remote_snapshot_timestamp":1771789905,"replay_state":"idle"} 2026-02-22T19:52:06.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:06.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:06.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:06.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:06.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:06.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:06.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:51:58' 2026-02-22T19:52:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:51:58' 2026-02-22T19:52:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:51:58' 2026-02-22T19:52:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:06.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789905,"remote_snapshot_timestamp":1771789905,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:51:58test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789914,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:51:56' 2026-02-22T19:52:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:51:58' 2026-02-22T19:52:06.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test 2026-02-22T19:52:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T19:52:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T19:52:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.export 2026-02-22T19:52:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:52:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:52:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T19:52:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.export 2026-02-22T19:52:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:52:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:52:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:52:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:52:06.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:52:06.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:52:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:52:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:52:06.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:52:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T19:52:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: stop/start/restart mirror via admin socket' 2026-02-22T19:52:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T19:52:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T19:52:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 19:52:06 TEST: stop/start/restart mirror via admin socket 2026-02-22T19:52:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 19:52:06 TEST: stop/start/restart mirror via admin socket 2026-02-22T19:52:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-02-22T19:52:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:06.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:06.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-02-22T19:52:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-02-22T19:52:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-02-22T19:52:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:52:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:52:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:52:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:52:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:52:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:08.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:08.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:08.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:52:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:52:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:52:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T19:52:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:09.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:09.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:52:09.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:52:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:52:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:52:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: stopped 2026-02-22T19:52:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:07' 2026-02-22T19:52:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:52:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:52:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: stopped 2026-02-22T19:52:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:10.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:07' 2026-02-22T19:52:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:52:07' 2026-02-22T19:52:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:07' 2026-02-22T19:52:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+stoppedstopped43284339mirror.1trial0982026-02-22 19:52:07test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+stoppedstopped43244333mirror.0trial0982026-02-22 19:52:06' 2026-02-22T19:52:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:07' 2026-02-22T19:52:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:52:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:52:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T19:52:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:52:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: stopped 2026-02-22T19:52:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:06' 2026-02-22T19:52:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:52:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:52:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:11.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: stopped 2026-02-22T19:52:11.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:11.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:06' 2026-02-22T19:52:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 19:52:06' 2026-02-22T19:52:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:06' 2026-02-22T19:52:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+stoppedstopped43284339mirror.1trial0982026-02-22 19:52:07test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+stoppedstopped43244333mirror.0trial0982026-02-22 19:52:06' 2026-02-22T19:52:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:06' 2026-02-22T19:52:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:52:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:52:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror start 2026-02-22T19:52:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror start 2026-02-22T19:52:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:11.605 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:11.605 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:11.605 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:11.605 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start 2026-02-22T19:52:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror start 2026-02-22T19:52:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:11.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:11.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:11.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:11.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:11.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start 2026-02-22T19:52:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror start 2026-02-22T19:52:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:11.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:11.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start 2026-02-22T19:52:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror start 2026-02-22T19:52:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:11.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:11.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:11.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:11.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start 2026-02-22T19:52:11.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:11.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:11.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:52:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:12.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:12.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:12.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:12.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:12.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:12.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:12.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:52:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:52:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:15.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:11' 2026-02-22T19:52:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:11' 2026-02-22T19:52:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:52:11' 2026-02-22T19:52:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:11' 2026-02-22T19:52:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:15.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:11test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:11' 2026-02-22T19:52:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:11' 2026-02-22T19:52:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:15.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T19:52:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T19:52:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:16.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:16.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:16.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:16.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:16.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:11' 2026-02-22T19:52:16.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:16.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:16.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:16.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:16.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:16.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:16.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:16.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:11' 2026-02-22T19:52:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 19:52:11' 2026-02-22T19:52:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:11' 2026-02-22T19:52:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:11test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:11' 2026-02-22T19:52:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:11' 2026-02-22T19:52:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:16.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:16.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-02-22T19:52:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-02-22T19:52:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-02-22T19:52:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-02-22T19:52:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-02-22T19:52:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-02-22T19:52:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-02-22T19:52:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-02-22T19:52:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:16.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:52:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:17.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:17.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:17.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:17.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:17.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:17.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:17.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:17.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:17.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:17.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:17.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:17.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:17.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:17.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:17.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:18.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:18.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:18.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:18.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:18.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:18.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:18.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:18.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:18.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:18.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:18.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:18.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:18.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:18.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:18.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:18.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:18.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:52:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:19.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:52:19.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:19.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:19.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:19.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:19.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:19.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:19.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:16' 2026-02-22T19:52:19.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:19.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:19.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:16' 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:52:16' 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:16' 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:16test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:16' 2026-02-22T19:52:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:19.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:16' 2026-02-22T19:52:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T19:52:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:19.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:20.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T19:52:20.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:20.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:20.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:20.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:20.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:20.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:20.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:20.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:20.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:16' 2026-02-22T19:52:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:20.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:20.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:20.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:20.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:20.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:20.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:20.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:20.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:16' 2026-02-22T19:52:20.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 19:52:16' 2026-02-22T19:52:20.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:20.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:20.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:20.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:20.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:16' 2026-02-22T19:52:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:16test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:16' 2026-02-22T19:52:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:16' 2026-02-22T19:52:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:20.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:20.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:20.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-02-22T19:52:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-02-22T19:52:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-02-22T19:52:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-02-22T19:52:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-02-22T19:52:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-02-22T19:52:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-02-22T19:52:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-02-22T19:52:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:52:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:52:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:52:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:52:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:52:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:22.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:22.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:22.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:52:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:52:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T19:52:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:52:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:23.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:23.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:23.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:52:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:52:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:23.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:52:23.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:23.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:52:23.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:23.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:23.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:23.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:52:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:24.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:24.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:24.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:52:24.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: stopped 2026-02-22T19:52:24.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:24.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:21' 2026-02-22T19:52:24.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:52:24.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:52:24.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:24.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: stopped 2026-02-22T19:52:24.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:24.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:24.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:24.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:24.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:24.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:24.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:24.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:24.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:21' 2026-02-22T19:52:24.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:52:21' 2026-02-22T19:52:24.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:24.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:24.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:24.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:24.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:21' 2026-02-22T19:52:24.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:24.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:24.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:24.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+stoppedstopped43284339mirror.1trial0982026-02-22 19:52:21test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+stoppedstopped43244333mirror.0trial0982026-02-22 19:52:21' 2026-02-22T19:52:24.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:24.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:24.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:21' 2026-02-22T19:52:24.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:24.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:52:24.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:24.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:52:24.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T19:52:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T19:52:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:52:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: stopped 2026-02-22T19:52:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:21' 2026-02-22T19:52:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:52:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:52:25.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:25.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: stopped 2026-02-22T19:52:25.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:25.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:25.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:21' 2026-02-22T19:52:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 19:52:21' 2026-02-22T19:52:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:21' 2026-02-22T19:52:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+stoppedstopped43284339mirror.1trial0982026-02-22 19:52:21test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+stoppedstopped43244333mirror.0trial0982026-02-22 19:52:21' 2026-02-22T19:52:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:25.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:21' 2026-02-22T19:52:25.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:25.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:52:25.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:25.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:52:25.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:25.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:25.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:25.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-02-22T19:52:25.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:25.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:25.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:25.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-02-22T19:52:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-02-22T19:52:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-02-22T19:52:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-02-22T19:52:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:25.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:25.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:25.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:25.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:25.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:25.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:25.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:25.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-02-22T19:52:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-02-22T19:52:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-02-22T19:52:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:52:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:52:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:52:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:27.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:27.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:27.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:27.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:27.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:27.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:27.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:27.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:28.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:28.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:28.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:28.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:28.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:28.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:28.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:28.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:28.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:28.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:28.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:28.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:28.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:28.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:28.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:28.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:28.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:28.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:52:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:25' 2026-02-22T19:52:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:29.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:29.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:25' 2026-02-22T19:52:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:52:25' 2026-02-22T19:52:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:25' 2026-02-22T19:52:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:29.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:29.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:29.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:25test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:25' 2026-02-22T19:52:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:25' 2026-02-22T19:52:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T19:52:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T19:52:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:25' 2026-02-22T19:52:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:25' 2026-02-22T19:52:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 19:52:25' 2026-02-22T19:52:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:25' 2026-02-22T19:52:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:25test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:25' 2026-02-22T19:52:30.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:30.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:25' 2026-02-22T19:52:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:30.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:30.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:30.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:30.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:30.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:30.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:30.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:30.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:30.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:30.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:30.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:30.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:30.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:30.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:30.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:31.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:52:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:52:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:31.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:31.710 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:31.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:31.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:31.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:52:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:52:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:52:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T19:52:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T19:52:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:52:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T19:52:32.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:32.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:32.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:32.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:52:32.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:32.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:32.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:32.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:52:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:52:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:52:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:52:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:52:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: stopped 2026-02-22T19:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:30' 2026-02-22T19:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:52:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: stopped 2026-02-22T19:52:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:30' 2026-02-22T19:52:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:52:30' 2026-02-22T19:52:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:30' 2026-02-22T19:52:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+stoppedstopped43284339mirror.1trial0982026-02-22 19:52:30test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+stoppedstopped43244333mirror.0trial0982026-02-22 19:52:30' 2026-02-22T19:52:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:30' 2026-02-22T19:52:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:52:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:52:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T19:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T19:52:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: stopped 2026-02-22T19:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:30' 2026-02-22T19:52:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:52:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:52:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: stopped 2026-02-22T19:52:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:30' 2026-02-22T19:52:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 19:52:30' 2026-02-22T19:52:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:30' 2026-02-22T19:52:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+stoppedstopped43284339mirror.1trial0982026-02-22 19:52:30test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+stoppedstopped43244333mirror.0trial0982026-02-22 19:52:30' 2026-02-22T19:52:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:30' 2026-02-22T19:52:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:52:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:52:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-22T19:52:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_instance=cluster1 2026-02-22T19:52:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance=cluster1 2026-02-22T19:52:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loop_instance 2026-02-22T19:52:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:52:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-22T19:52:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:35.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-22T19:52:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror start mirror/test1 2026-02-22T19:52:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror/test 2026-02-22T19:52:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror/test 2026-02-22T19:52:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:52:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:52:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:52:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:36.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:36.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:36.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:36.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:36.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:36.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:36.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:36.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:36.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:36.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:36.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:36.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:36.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:52:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:52:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:37.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:37.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:35' 2026-02-22T19:52:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:35' 2026-02-22T19:52:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:52:35' 2026-02-22T19:52:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:35' 2026-02-22T19:52:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:35test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+stoppedstopped43244333mirror.0trial0982026-02-22 19:52:30' 2026-02-22T19:52:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:35' 2026-02-22T19:52:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:37.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:37.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:37.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror start mirror cluster2-DNE 2026-02-22T19:52:37.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:37.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:37.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror cluster2-DNE 2026-02-22T19:52:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:37.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror cluster2-DNE 2026-02-22T19:52:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror cluster2-DNE 2026-02-22T19:52:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:37.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:37.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:37.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:37.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror cluster2-DNE 2026-02-22T19:52:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror start mirror cluster2-DNE 2026-02-22T19:52:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:37.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:37.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:37.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start mirror cluster2-DNE 2026-02-22T19:52:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror start mirror cluster2-DNE 2026-02-22T19:52:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start mirror cluster2-DNE 2026-02-22T19:52:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T19:52:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:37.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:37.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:37.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:37.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:37.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:37.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:37.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:38.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:38.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:38.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:38.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:38.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:38.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:38.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T19:52:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:39.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:39.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:39.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:37' 2026-02-22T19:52:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:39.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:37' 2026-02-22T19:52:39.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 19:52:37' 2026-02-22T19:52:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:37' 2026-02-22T19:52:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:35test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:37' 2026-02-22T19:52:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:37' 2026-02-22T19:52:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test 2026-02-22T19:52:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_instance=cluster1 2026-02-22T19:52:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance=cluster1 2026-02-22T19:52:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loop_instance 2026-02-22T19:52:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:52:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test 2026-02-22T19:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test 2026-02-22T19:52:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror restart mirror/test1 2026-02-22T19:52:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror/test 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror/test 2026-02-22T19:52:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:52:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:41.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:41.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:41.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:41.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:41.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:41.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:41.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:41.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:41.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:41.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:52:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:52:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:40' 2026-02-22T19:52:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:40' 2026-02-22T19:52:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:52:40' 2026-02-22T19:52:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:40' 2026-02-22T19:52:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:40test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:37' 2026-02-22T19:52:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:40' 2026-02-22T19:52:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:42.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:42.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:42.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:42.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:42.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:42.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:42.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:42.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:42.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:42.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:42.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:42.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:42.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:42.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:42.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:42.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:42.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:42.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:42.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:42.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:42.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:42.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:42.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:42.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:42.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:42.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:42.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:52:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:43.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:44.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:44.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:44.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:44.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:44.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:44.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:44.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:44.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T19:52:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:52:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:52:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:52:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:52:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:52:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:52:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:52:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:52:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:47.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T19:52:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:52:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:47.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:47.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:47.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:47.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:47.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:47.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:47.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:47.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:47.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:47.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:47.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:47.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:47.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:47.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:47.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:47.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:52:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:52:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:52:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:48.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:48.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:52:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: stopped 2026-02-22T19:52:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:45' 2026-02-22T19:52:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:52:48.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:52:48.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:48.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: stopped 2026-02-22T19:52:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:48.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:48.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:48.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:45' 2026-02-22T19:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:52:45' 2026-02-22T19:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:45' 2026-02-22T19:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:48.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+stoppedstopped43284339mirror.1trial0982026-02-22 19:52:45test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+stoppedstopped43244333mirror.0trial0982026-02-22 19:52:44' 2026-02-22T19:52:48.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:45' 2026-02-22T19:52:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:48.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:52:48.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:48.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:52:48.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:49.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T19:52:49.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:49.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:49.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:49.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:49.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:52:49.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: stopped 2026-02-22T19:52:49.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:49.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:44' 2026-02-22T19:52:49.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:52:49.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:52:49.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:49.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: stopped 2026-02-22T19:52:49.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:49.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:49.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:49.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:49.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:44' 2026-02-22T19:52:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 19:52:44' 2026-02-22T19:52:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:52:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:44' 2026-02-22T19:52:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+stoppedstopped43284339mirror.1trial0982026-02-22 19:52:45test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+stoppedstopped43244333mirror.0trial0982026-02-22 19:52:44' 2026-02-22T19:52:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:49.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:44' 2026-02-22T19:52:49.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:52:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:52:49.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:52:49.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:52:49.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:49.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:49.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:49.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:49.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:49.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:49.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:49.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:49.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:49.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:49.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:49.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:49.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:49.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:49.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:49.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:49.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:49.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T19:52:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:52:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:52:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:52:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:52:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:52:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:no valid command found; 1 closest matches: 2026-02-22T19:52:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status mirror/test1 2026-02-22T19:52:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:50.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:52:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:51.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:51.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:51.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T19:52:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T19:52:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T19:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:52.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:52:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:52.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:52.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:52.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:52:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:53.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:53.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:53.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:53.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:52:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:52:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:49' 2026-02-22T19:52:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:53.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:53.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:49' 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:52:49' 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:49' 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:53.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:49test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:49' 2026-02-22T19:52:53.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:53.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:52:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:49' 2026-02-22T19:52:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:52:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T19:52:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:54.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:54.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T19:52:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test1: 2026-02-22T19:52:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: ef0d415f-0391-42f8-b805-2bf223ba80c9 2026-02-22T19:52:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:52:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:54' 2026-02-22T19:52:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"} 2026-02-22T19:52:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:54.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:52:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:54' 2026-02-22T19:52:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 19:52:54' 2026-02-22T19:52:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:54' 2026-02-22T19:52:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:54test1ef0d415f-0391-42f8-b805-2bf223ba80c9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789917,"remote_snapshot_timestamp":1771789917,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:52:54' 2026-02-22T19:52:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T19:52:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:54' 2026-02-22T19:52:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T19:52:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:54.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ flush cluster1 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool= 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image= 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemons cluster1 rbd mirror flush 2026-02-22T19:52:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_instance=cluster1 2026-02-22T19:52:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance=cluster1 2026-02-22T19:52:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loop_instance 2026-02-22T19:52:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:52:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror flush 2026-02-22T19:52:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush 2026-02-22T19:52:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ all_admin_daemons cluster1 rbd mirror status 2026-02-22T19:52:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror status 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status 2026-02-22T19:52:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:{ 2026-02-22T19:52:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool_replayers": [ 2026-02-22T19:52:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "peer": "uuid: df7543bc-b569-4771-a1ad-0eef6af1c1d2 cluster: cluster2-DNE client: client.mirror", 2026-02-22T19:52:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool": "mirror", 2026-02-22T19:52:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instance_id": "4333", 2026-02-22T19:52:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "stopped (manual)", 2026-02-22T19:52:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader_instance_id": "4333", 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader": true, 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instances": [ 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "4333", 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "4339", 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "4344", 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "4346" 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ], 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.21670.cluster1.94001024522600.asok", 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.21670.cluster2-DNE.94001037318504.asok", 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "sync_throttler": { 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 5, 2026-02-22T19:52:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "deletion_throttler": { 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 1, 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [ 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "mirror/test1", 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ], 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter": { 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter_status": { 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "delete_images_queue": [], 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "failed_deletes_queue": [] 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "namespaces": [ 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns1", 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [], 2026-02-22T19:52:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter": { 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter_status": { 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "delete_images_queue": [], 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "failed_deletes_queue": [] 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns2", 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [], 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter": { 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter_status": { 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "delete_images_queue": [], 2026-02-22T19:52:54.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "failed_deletes_queue": [] 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "peer": "uuid: c1b1b313-242d-44ac-bb04-cf62de357d26 cluster: cluster2-DNE client: client.mirror", 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool": "mirror_parent", 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instance_id": "4349", 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "stopped (manual)", 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader_instance_id": "4349", 2026-02-22T19:52:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader": true, 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instances": [ 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "4349", 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "4352", 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "4358", 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "4359" 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ], 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.21670.cluster1.94001036622184.asok", 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.21670.cluster2-DNE.94001026316648.asok", 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "sync_throttler": { 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 5, 2026-02-22T19:52:54.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "deletion_throttler": { 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 1, 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [], 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter": { 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter_status": { 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "delete_images_queue": [], 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "failed_deletes_queue": [] 2026-02-22T19:52:54.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "namespaces": [ 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns1", 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [], 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter": { 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_deleter_status": { 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "delete_images_queue": [], 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "failed_deletes_queue": [] 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:} 2026-02-22T19:52:54.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:54.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror status 2026-02-22T19:52:54.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:54.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:54.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:54.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:54.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:54.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status 2026-02-22T19:52:54.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:{ 2026-02-22T19:52:54.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool_replayers": [ 2026-02-22T19:52:54.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "peer": "uuid: df7543bc-b569-4771-a1ad-0eef6af1c1d2 cluster: cluster2-DNE client: client.mirror", 2026-02-22T19:52:54.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool": "mirror", 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instance_id": "4339", 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "stopped (manual)", 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader_instance_id": "4333", 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader": false, 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.21698.cluster1.94085042665832.asok", 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.21698.cluster2-DNE.94085055543656.asok", 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "sync_throttler": { 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 5, 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "deletion_throttler": { 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 1, 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [ 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "mirror/test", 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ], 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "namespaces": [ 2026-02-22T19:52:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns1", 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [] 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns2", 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [] 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "peer": "uuid: c1b1b313-242d-44ac-bb04-cf62de357d26 cluster: cluster2-DNE client: client.mirror", 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool": "mirror_parent", 2026-02-22T19:52:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instance_id": "4352", 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "stopped (manual)", 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader_instance_id": "4349", 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader": false, 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.21698.cluster1.94085043124584.asok", 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.21698.cluster2-DNE.94085045098856.asok", 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "sync_throttler": { 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 5, 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "deletion_throttler": { 2026-02-22T19:52:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 1, 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [], 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "namespaces": [ 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns1", 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [] 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:} 2026-02-22T19:52:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:2 rbd mirror status 2026-02-22T19:52:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T19:52:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:52:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T19:52:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:52:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:52:54.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:54.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:52:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status 2026-02-22T19:52:54.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:{ 2026-02-22T19:52:54.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool_replayers": [ 2026-02-22T19:52:54.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "peer": "uuid: df7543bc-b569-4771-a1ad-0eef6af1c1d2 cluster: cluster2-DNE client: client.mirror", 2026-02-22T19:52:54.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool": "mirror", 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instance_id": "4344", 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "stopped (manual)", 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader_instance_id": "4333", 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader": false, 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.21742.cluster1.94870364313960.asok", 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.21742.cluster2-DNE.94870377216360.asok", 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "sync_throttler": { 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 5, 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "deletion_throttler": { 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 1, 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [], 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "namespaces": [ 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns1", 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [] 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns2", 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [] 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "peer": "uuid: c1b1b313-242d-44ac-bb04-cf62de357d26 cluster: cluster2-DNE client: client.mirror", 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool": "mirror_parent", 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instance_id": "4359", 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "stopped (manual)", 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader_instance_id": "4349", 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader": false, 2026-02-22T19:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.21742.cluster1.94870364707176.asok", 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.21742.cluster2-DNE.94870366656872.asok", 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "sync_throttler": { 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 5, 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "deletion_throttler": { 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 1, 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [], 2026-02-22T19:52:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "namespaces": [ 2026-02-22T19:52:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns1", 2026-02-22T19:52:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [] 2026-02-22T19:52:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:} 2026-02-22T19:52:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:3 rbd mirror status 2026-02-22T19:52:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T19:52:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:52:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T19:52:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:52:54.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:52:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:52:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status 2026-02-22T19:52:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:{ 2026-02-22T19:52:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool_replayers": [ 2026-02-22T19:52:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "peer": "uuid: df7543bc-b569-4771-a1ad-0eef6af1c1d2 cluster: cluster2-DNE client: client.mirror", 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool": "mirror", 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instance_id": "4346", 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "stopped (manual)", 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader_instance_id": "4333", 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader": false, 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.21802.cluster1.94354084600168.asok", 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.21802.cluster2-DNE.94354097437032.asok", 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "sync_throttler": { 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 5, 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "deletion_throttler": { 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 1, 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [], 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "namespaces": [ 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns1", 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [] 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns2", 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [] 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "peer": "uuid: c1b1b313-242d-44ac-bb04-cf62de357d26 cluster: cluster2-DNE client: client.mirror", 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "pool": "mirror_parent", 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "instance_id": "4358", 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "stopped (manual)", 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader_instance_id": "4349", 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "leader": false, 2026-02-22T19:52:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.21802.cluster1.94354085034344.asok", 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.21802.cluster2-DNE.94354087123304.asok", 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "sync_throttler": { 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 5, 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "deletion_throttler": { 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "max_parallel_requests": 1, 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "running_requests": 0, 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "waiting_requests": 0 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: }, 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [], 2026-02-22T19:52:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "namespaces": [ 2026-02-22T19:52:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: { 2026-02-22T19:52:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "ns1", 2026-02-22T19:52:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "image_replayers": [] 2026-02-22T19:52:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: } 2026-02-22T19:52:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: ] 2026-02-22T19:52:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:} 2026-02-22T19:52:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test1 2026-02-22T19:52:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:52:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:52:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test1 2026-02-22T19:52:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test1 2026-02-22T19:52:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test1 2026-02-22T19:52:54.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test1 2026-02-22T19:52:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T19:52:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: test image rename' 2026-02-22T19:52:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T19:52:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T19:52:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 19:52:55 TEST: test image rename 2026-02-22T19:52:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 19:52:55 TEST: test image rename 2026-02-22T19:52:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ new_name=test_RENAMED 2026-02-22T19:52:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_image cluster2 mirror test test_RENAMED 2026-02-22T19:52:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:52:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_name=test_RENAMED 2026-02-22T19:52:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rename mirror/test mirror/test_RENAMED 2026-02-22T19:52:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:52:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_RENAMED 2026-02-22T19:52:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:52:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:52:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_RENAMED 2026-02-22T19:52:57.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 8 2026-02-22T19:52:57.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-02-22T19:52:57.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:57.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:57.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:52:57.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T19:52:57.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:52:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T19:52:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:52:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:52:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:52:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:52:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:52:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:52:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:52:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:52:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:52:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:52:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:52:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:52:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:52:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:58.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:58.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:58.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T19:52:58.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:52:58.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:52:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:52:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:52:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:52:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:52:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:52:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:52:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:58.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:58.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:58.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:58.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:52:58.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:52:58.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:52:58.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:52:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:58.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:52:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-02-22T19:52:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:52:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:52:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:52:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-02-22T19:52:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_RENAMED: 2026-02-22T19:52:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:52:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:52:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:52:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:54' 2026-02-22T19:52:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:52:59.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:52:59.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:52:59.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"} 2026-02-22T19:52:59.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:52:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:52:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:52:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:52:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:52:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:52:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:54' 2026-02-22T19:52:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-02-22 19:52:54' 2026-02-22T19:52:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:52:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:52:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:54' 2026-02-22T19:52:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:52:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:52:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:52:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_RENAMED1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789924,"remote_snapshot_timestamp":1771789924,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:54' 2026-02-22T19:52:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:52:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-02-22T19:52:59.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:54' 2026-02-22T19:52:59.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-02-22T19:52:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:52:59.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:52:59.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:52:59.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:52:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_instance=cluster1 2026-02-22T19:52:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance=cluster1 2026-02-22T19:52:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loop_instance 2026-02-22T19:52:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:59.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:59.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:59.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:59.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T19:52:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:52:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:59.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:59.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:59.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-22T19:52:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:{ 2026-02-22T19:52:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "name": "mirror/test_RENAMED", 2026-02-22T19:52:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:52:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:} 2026-02-22T19:52:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test_RENAMED 2026-02-22T19:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_instance=cluster1 2026-02-22T19:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance=cluster1 2026-02-22T19:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loop_instance 2026-02-22T19:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test_RENAMED 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:52:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:52:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:52:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:52:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:52:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:52:59.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:59.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:59.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:52:59.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test_RENAMED 2026-02-22T19:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T19:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror restart : restart rbd mirror 2026-02-22T19:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:52:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:52:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror/test_RENAMED 2026-02-22T19:52:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T19:52:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:52:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T19:52:59.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:52:59.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:52:59.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:52:59.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:52:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror/test_RENAMED 2026-02-22T19:52:59.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:52:59.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:52:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:00.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T19:53:00.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:00.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:00.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:53:00.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:00.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:00.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:00.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-22T19:53:00.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:00.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:53:00.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:00.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:00.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:53:00.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:00.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:00.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T19:53:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T19:53:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:53:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:53:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:53:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:53:00.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:53:00.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-22T19:53:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:00.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:53:00.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:53:00.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:00.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:53:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:53:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:53:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-02-22T19:53:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:53:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-02-22T19:53:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:53:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-02-22T19:53:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_RENAMED: 2026-02-22T19:53:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 1022b5fb-140c-4e47-a45c-827e5aa56e92 2026-02-22T19:53:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:53:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789976,"remote_snapshot_timestamp":1771789976,"replay_state":"idle"} 2026-02-22T19:53:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T19:53:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:52:59' 2026-02-22T19:53:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:53:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:53:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789976,"remote_snapshot_timestamp":1771789976,"replay_state":"idle"} 2026-02-22T19:53:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:53:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:53:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T19:53:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:01.936 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:59' 2026-02-22T19:53:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-02-22 19:52:59' 2026-02-22T19:53:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:53:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:52:59' 2026-02-22T19:53:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_RENAMED1022b5fb-140c-4e47-a45c-827e5aa56e92up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789976,"remote_snapshot_timestamp":1771789976,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 19:52:59' 2026-02-22T19:53:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-02-22T19:53:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:52:59' 2026-02-22T19:53:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-02-22T19:53:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:53:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:53:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:53:01.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:53:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_image cluster2 mirror test_RENAMED test 2026-02-22T19:53:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_RENAMED 2026-02-22T19:53:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_name=test 2026-02-22T19:53:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rename mirror/test_RENAMED mirror/test 2026-02-22T19:53:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:53:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:53:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:53:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 9 2026-02-22T19:53:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:53:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:53:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:53:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:53:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:53:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:53:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:53:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:04.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:04.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:53:04.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:04.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:53:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:53:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:53:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:53:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:53:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:53:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:53:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:53:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:04.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:53:04.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:53:04.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:53:04.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:04.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: test trash move restore' 2026-02-22T19:53:04.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T19:53:04.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T19:53:04.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 19:53:04 TEST: test trash move restore 2026-02-22T19:53:04.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 19:53:04 TEST: test trash move restore 2026-02-22T19:53:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster2 mirror test 2026-02-22T19:53:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T19:53:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster2 info mirror/test 2026-02-22T19:53:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T19:53:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id=10b5b5178dbe 2026-02-22T19:53:04.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ trash_move cluster2 mirror test 2026-02-22T19:53:04.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:04.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:04.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:04.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 trash move mirror/test 2026-02-22T19:53:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-22T19:53:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T19:53:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T19:53:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:53:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T19:53:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T19:53:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T19:53:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T19:53:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T19:53:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T19:53:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T19:53:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T19:53:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-02-22T19:53:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T19:53:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T19:53:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T19:53:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T19:53:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T19:53:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T19:53:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T19:53:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T19:53:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T19:53:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T19:53:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:53:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T19:53:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ trash_restore cluster2 mirror test 10b5b5178dbe snapshot 2026-02-22T19:53:05.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:05.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:05.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:05.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=10b5b5178dbe 2026-02-22T19:53:05.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T19:53:05.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 trash restore mirror/10b5b5178dbe 2026-02-22T19:53:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T19:53:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T19:53:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-22T19:53:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T19:53:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-22T19:53:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T19:53:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-22T19:53:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test': 2026-02-22T19:53:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T19:53:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T19:53:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T19:53:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 10b5b5178dbe 2026-02-22T19:53:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.10b5b5178dbe 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 19:51:41 2026 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 19:51:41 2026 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 19:51:41 2026 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 2a0ba85a-7ae0-4fcd-9dd3-c2bca2283b6b 2026-02-22T19:53:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T19:53:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:53:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:53:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:53:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:53:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:53:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:53:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:53:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:07.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:07.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:07.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:07.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:07.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:53:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:07.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:53:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:53:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:53:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:53:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:53:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:53:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:53:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:53:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:53:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:53:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T19:53:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:53:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:53:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:53:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:53:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T19:53:07.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:53:07.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:07.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T19:53:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T19:53:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:53:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:07.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T19:53:07.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T19:53:07.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:53:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:53:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:53:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:53:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T19:53:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:53:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T19:53:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:08.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:08.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:08.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:08.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:08.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:08.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:53:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:53:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:53:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:53:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:53:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed (with rbd-mirror on one cluster)' 2026-02-22T19:53:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T19:53:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T19:53:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 19:53:08 TEST: check if removed 'images'\''' OMAP are removed '(with' rbd-mirror on one 'cluster)' 2026-02-22T19:53:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 19:53:08 TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster) 2026-02-22T19:53:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test 2026-02-22T19:53:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T19:53:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:53:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test 2026-02-22T19:53:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-22T19:53:08.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-22T19:53:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T19:53:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-22T19:53:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-22T19:53:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirroring 2026-02-22T19:53:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=status_global 2026-02-22T19:53:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T19:53:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:53:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:53:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=status_global 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c status_global 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T19:53:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T19:53:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T19:53:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:53:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:53:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T19:53:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-22T19:53:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirroring 2026-02-22T19:53:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=image_ 2026-02-22T19:53:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T19:53:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:53:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:53:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-22T19:53:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T19:53:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_ 2026-02-22T19:53:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T19:53:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_ 2026-02-22T19:53:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T19:53:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T19:53:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T19:53:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:53:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:53:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T19:53:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-22T19:53:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T19:53:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=image_map 2026-02-22T19:53:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T19:53:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:53:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:53:09.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T19:53:09.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:09.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T19:53:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_map 2026-02-22T19:53:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T19:53:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_map 2026-02-22T19:53:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T19:53:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T19:53:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 1 = 0 2026-02-22T19:53:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=1 2026-02-22T19:53:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:53:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T19:53:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T19:53:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:53:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T19:53:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T19:53:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_map 2026-02-22T19:53:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T19:53:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_map 2026-02-22T19:53:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T19:53:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T19:53:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 1 = 0 2026-02-22T19:53:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=1 2026-02-22T19:53:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:53:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T19:53:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T19:53:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:53:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:53:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_map 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_map 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T19:53:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T19:53:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T19:53:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:53:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:53:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T19:53:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-22T19:53:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-22T19:53:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirroring 2026-02-22T19:53:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=status_global 2026-02-22T19:53:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T19:53:12.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:53:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:53:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-22T19:53:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T19:53:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=status_global 2026-02-22T19:53:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T19:53:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c status_global 2026-02-22T19:53:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T19:53:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T19:53:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T19:53:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirroring 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=image_ 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T19:53:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_ 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_ 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T19:53:12.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T19:53:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T19:53:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:53:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:53:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T19:53:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-22T19:53:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T19:53:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=image_map 2026-02-22T19:53:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T19:53:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T19:53:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T19:53:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-22T19:53:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T19:53:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_map 2026-02-22T19:53:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-22T19:53:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T19:53:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T19:53:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T19:53:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_map 2026-02-22T19:53:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:error getting omap key set mirror/rbd_mirror_leader: (2) No such file or directory 2026-02-22T19:53:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T19:53:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T19:53:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T19:53:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T19:53:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:53:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:53:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T19:53:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T19:53:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test 2026-02-22T19:53:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T19:53:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T19:53:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-22T19:53:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T19:53:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-22T19:53:13.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T19:53:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-22T19:53:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test': 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 114815b984ce 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.114815b984ce 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 19:53:12 2026 2026-02-22T19:53:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 19:53:12 2026 2026-02-22T19:53:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 19:53:12 2026 2026-02-22T19:53:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T19:53:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T19:53:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T19:53:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:53:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:53:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:53:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:53:14.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:14.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:14.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:14.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:53:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T19:53:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T19:53:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T19:53:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T19:53:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T19:53:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T19:53:14.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T19:53:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T19:53:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T19:53:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:53:14.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:14.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T19:53:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T19:53:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T19:53:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T19:53:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:53:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T19:53:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T19:53:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:53:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T19:53:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T19:53:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T19:53:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T19:53:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T19:53:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T19:53:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T19:53:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T19:53:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T19:53:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:53:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:53:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T19:53:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T19:53:14.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:53:14.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:14.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T19:53:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T19:53:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T19:53:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T19:53:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T19:53:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T19:53:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:53:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:53:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T19:53:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T19:53:14.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T19:53:14.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T19:53:14.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T19:53:14.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:14.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T19:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:53:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:53:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:15.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test 100 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T19:53:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T19:53:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T19:53:15.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T19:53:15.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.49 bytes/sec: 33 MiB/s 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:53:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 12 2026-02-22T19:53:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:53:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:53:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:53:16.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:53:16.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:53:16.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:53:16.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:53:16.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:53:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:53:16.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:16.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:53:16.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:16.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:16.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:16.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:53:16.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:53:16.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:53:16.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:53:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=12 2026-02-22T19:53:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:53:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:53:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:53:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:53:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:53:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:53:16.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=11 2026-02-22T19:53:16.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 12 = 11 2026-02-22T19:53:16.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:53:16.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T19:53:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:53:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:53:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=12 2026-02-22T19:53:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 12 = 12 2026-02-22T19:53:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:53:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:53:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789993,"replay_state":"idle"} 2026-02-22T19:53:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:53:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:15' 2026-02-22T19:53:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:53:17.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:53:17.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789993,"replay_state":"idle"} 2026-02-22T19:53:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:53:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:17.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:53:17.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:53:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:15' 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:53:15' 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:15' 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:17.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771789993,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:53:15' 2026-02-22T19:53:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:53:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:15' 2026-02-22T19:53:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:53:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:53:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:53:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:53:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:53:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:17.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: failover and failback' 2026-02-22T19:53:17.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T19:53:17.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T19:53:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 19:53:17 TEST: failover and failback 2026-02-22T19:53:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 19:53:17 TEST: failover and failback 2026-02-22T19:53:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster2 2026-02-22T19:53:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T19:53:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster2:0 2026-02-22T19:53:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:0 2026-02-22T19:53:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:53:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster2:0 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster2 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster2 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:53:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:53:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster2:1 2026-02-22T19:53:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:1 2026-02-22T19:53:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:53:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T19:53:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster2:1 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster2 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster2 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:53:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:53:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster2:2 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:2 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster2:2 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster2 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T19:53:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster2 2026-02-22T19:53:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:53:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T19:53:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T19:53:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:53:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:53:18.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:18.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster2:3 2026-02-22T19:53:18.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:3 2026-02-22T19:53:18.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T19:53:18.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T19:53:18.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster2:3 2026-02-22T19:53:18.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster2 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster2 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:53:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T19:53:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:53:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:53:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:53:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:53:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:53:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:53:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:53:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:53:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:53:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:53:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:21.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:21.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:21.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:21.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:53:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:19 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:20' 2026-02-22T19:53:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:53:21.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:53:21.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:53:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:21.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:21.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:21.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:21.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:53:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:21.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:19' 2026-02-22T19:53:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:53:19' 2026-02-22T19:53:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:19' 2026-02-22T19:53:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 19:53:19cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:53:20' 2026-02-22T19:53:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:53:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:19' 2026-02-22T19:53:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:53:21.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:53:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:53:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:53:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:53:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:53:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:53:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:20 2026-02-22T19:53:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:53:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:53:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:53:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:19' 2026-02-22T19:53:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:53:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:53:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:22.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:53:22.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:22.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:22.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:22.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:22.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T19:53:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:53:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:53:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:53:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:53:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:53:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:23 2026-02-22T19:53:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:53:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:53:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:53:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:24' 2026-02-22T19:53:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:53:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:53:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:53:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:24.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:24.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:24.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:53:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:23' 2026-02-22T19:53:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:53:23' 2026-02-22T19:53:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:23' 2026-02-22T19:53:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:53:23cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:53:24' 2026-02-22T19:53:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:24.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:53:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:23' 2026-02-22T19:53:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:53:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:53:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:53:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:53:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:53:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:53:26.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:53:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:53:26.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:53:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:53:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:27.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:53:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:27.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:27.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:53:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:53:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:53:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:27.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:53:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:53:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:53:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:53:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:53:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:29.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:29.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:29.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:53:29.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:53:29.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:29.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test 100 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T19:53:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:53:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T19:53:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T19:53:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T19:53:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T19:53:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.49 bytes/sec: 33 MiB/s 2026-02-22T19:53:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:53:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:53:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:53:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:53:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:53:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:53:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 15 2026-02-22T19:53:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:53:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:53:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:53:31.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:53:31.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:53:31.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:53:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:53:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:53:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:53:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:53:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:53:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:53:31.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:53:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:53:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=15 2026-02-22T19:53:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:53:31.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:53:31.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:53:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:53:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:53:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:53:31.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=15 2026-02-22T19:53:31.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 15 = 15 2026-02-22T19:53:31.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:31.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:53:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:53:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:53:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:53:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:32.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:32.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:53:32.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:32.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:28 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789998,"remote_snapshot_timestamp":1771790005,"replay_state":"idle"} 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:29 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:53:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 14 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.cd7f4738-47a3-40a4-9373-c9b8cc6b1cf8 (peer_uuids:[]) 2026-02-22T19:53:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 15 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.99808f8d-e3e1-4b3a-9c31-63dbf75dd559 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:53:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:53:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:53:32.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:32.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:53:32.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:32.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:53:32.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:32.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:53:32.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:53:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:28' 2026-02-22T19:53:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:53:28' 2026-02-22T19:53:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:53:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:28' 2026-02-22T19:53:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:32.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:53:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789998,"remote_snapshot_timestamp":1771790005,"replay_state":"idle"}2026-02-22 19:53:29' 2026-02-22T19:53:32.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:32.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:53:32.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:28' 2026-02-22T19:53:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:53:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:53:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:53:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:53:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789998,"remote_snapshot_timestamp":1771790005,"replay_state":"idle"} 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:29 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:33' 2026-02-22T19:53:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:53:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:53:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789998,"remote_snapshot_timestamp":1771790005,"replay_state":"idle"} 2026-02-22T19:53:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:33.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:53:33.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:53:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:53:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:29' 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:53:29' 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:29' 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771789998,"remote_snapshot_timestamp":1771790005,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:53:29cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:53:33' 2026-02-22T19:53:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:53:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:29' 2026-02-22T19:53:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:53:33.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:53:33.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:53:33.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:53:33.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:53:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:33.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test 2026-02-22T19:53:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T19:53:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T19:53:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.export 2026-02-22T19:53:33.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:53:33.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:53:33.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T19:53:33.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.export 2026-02-22T19:53:33.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:53:33.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:53:33.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:53:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:53:33.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:53:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:53:33.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:53:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:53:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:53:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:53:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:53:35.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:53:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:53:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:53:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:35.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:53:35.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:35.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:35.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:53:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:53:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:53:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:53:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:53:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:36.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:36.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:36.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:36.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:36.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:53:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:53:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:37.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:53:37.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:37.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:37.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:37.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:37.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:37.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:37.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:37.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:37.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:53:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:53:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:53:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:35 2026-02-22T19:53:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:53:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:53:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:53:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:53:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:33' 2026-02-22T19:53:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:53:37.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:53:37.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:37.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:53:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:37.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:53:37.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:37.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:35' 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:53:35' 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:35' 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 19:53:35cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:53:33' 2026-02-22T19:53:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:53:37.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:35' 2026-02-22T19:53:37.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:53:37.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:53:37.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:37.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:53:37.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:53:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:53:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:53:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:38 2026-02-22T19:53:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:53:38.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:53:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:53:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:35' 2026-02-22T19:53:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:53:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:53:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:38.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:53:38.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:38.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:38' 2026-02-22T19:53:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:53:38' 2026-02-22T19:53:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:38' 2026-02-22T19:53:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:53:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 19:53:35' 2026-02-22T19:53:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:53:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:38' 2026-02-22T19:53:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:53:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:53:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:53:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:38.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:38.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:53:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:53:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:53:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:53:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:53:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:53:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:53:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:53:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:53:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:53:39.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:39.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:39.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:39.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:39.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:39.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:53:39.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:53:39.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:39.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:53:39.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:39.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:53:41.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:53:41.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:41.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:41.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:41.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:53:41.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:41.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:41.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:53:41.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:53:41.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:41.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:41.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:41.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:53:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:41.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:41.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:53:41.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:53:41.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:41.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:53:41.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:41.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T19:53:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:53:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:53:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:53:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:53:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:53:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:45.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:46.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:46.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:53:46.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:53:46.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:46.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:53:46.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:53:46.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:53:46.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:46.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:53:46.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:46.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:46.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:46.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:53:49.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:53:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:53:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:53:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:53:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:53:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:53:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:53:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:53:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:53:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:53:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:53:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:53:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:53:50.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:53:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:50.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:53:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:53:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:53:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:53:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:50.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:50.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:51.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:53:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:53:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:53:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:53:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:49 2026-02-22T19:53:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:53:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:53:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:53:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:48' 2026-02-22T19:53:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:53:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:53:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:51.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:53:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:51.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:53:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:49' 2026-02-22T19:53:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:53:49' 2026-02-22T19:53:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:49' 2026-02-22T19:53:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:53:49cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 19:53:48' 2026-02-22T19:53:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:53:51.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:49' 2026-02-22T19:53:51.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:53:51.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:53:51.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:53:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:53:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:53:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:53:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:53:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:53:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:53:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:48 2026-02-22T19:53:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:53:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:53:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:53:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:53:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:49' 2026-02-22T19:53:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:53:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:53:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:53:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:53:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:53:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:53:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:53:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:53:52.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:53:52.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:53:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:48' 2026-02-22T19:53:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:53:48' 2026-02-22T19:53:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:53:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:48' 2026-02-22T19:53:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:53:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:53:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:53:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted44434452mirror.0trial0982026-02-22 19:53:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:53:49' 2026-02-22T19:53:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:53:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:53:53.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:48' 2026-02-22T19:53:53.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:53:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:53:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:53:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:53:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:53:53.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:53.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:53.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:53:53.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:53.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:53:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:53:54.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:53:54.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:53:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:53:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:53:55.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:53:55.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:53:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:53:55.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:55.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:55.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:53:55.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:53:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:53:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:53:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:53:55.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:53:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:53:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:53:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:53:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:53:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:53:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:53:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:53:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:53:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:53:59.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 20 2026-02-22T19:53:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:53:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:53:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:53:59.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:53:59.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:53:59.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:53:59.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:53:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:53:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:53:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:53:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:53:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:53:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:53:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:53:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:53:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=20 2026-02-22T19:53:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:53:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:53:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:53:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:53:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:53:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:53:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:53:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=20 2026-02-22T19:53:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 20 = 20 2026-02-22T19:53:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:53:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T19:53:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:53:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:54:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:54:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:54:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790033,"replay_state":"idle"} 2026-02-22T19:54:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:54 2026-02-22T19:54:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:54:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:54:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:54:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:58' 2026-02-22T19:54:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:54:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:54:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790033,"replay_state":"idle"} 2026-02-22T19:54:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:54:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:54:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:54' 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:53:54' 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:54' 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:00.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790033,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:53:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:53:58' 2026-02-22T19:54:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:00.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:54' 2026-02-22T19:54:00.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:54:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:54:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:54:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:54:00.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:00.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:00.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:54:00.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:00.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:00.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:00.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:54:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:58 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790033,"replay_state":"idle"} 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:53:54 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 20 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.992b9721-3272-4082-9165-c30dd902cf9a (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:54:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:54:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:54:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:54:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:01.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:54:01.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:01.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:58' 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:53:58' 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:53:58' 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:53:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790033,"replay_state":"idle"}2026-02-22 19:53:54' 2026-02-22T19:54:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:53:58' 2026-02-22T19:54:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:54:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:54:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:54:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:54:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test 2026-02-22T19:54:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T19:54:01.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T19:54:01.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.export 2026-02-22T19:54:01.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:54:01.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T19:54:01.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.export 2026-02-22T19:54:01.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:01.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:01.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:54:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:54:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:54:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:01.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:54:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:01.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:01.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:54:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:54:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:54:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:54:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:54:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:54:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:54:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:54:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:54:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:54:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:54:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:54:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:54:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:03.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:03.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:54:03.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:54:03.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:03.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:54:03.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:54:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:54:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:54:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:54:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:54:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:05.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:05.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:05.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:54:05.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:05.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:05.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:54:05.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:54:05.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:05.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:54:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:05.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:05.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:54:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:54:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:54:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:04 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:03' 2026-02-22T19:54:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:54:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:54:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:54:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:06.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:06.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:06.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:04' 2026-02-22T19:54:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:54:04' 2026-02-22T19:54:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:04' 2026-02-22T19:54:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:54:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:54:03' 2026-02-22T19:54:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:06.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:04' 2026-02-22T19:54:06.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:54:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:54:06.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:54:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:54:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:54:07.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:07.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:03 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:04' 2026-02-22T19:54:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:54:07.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:54:07.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:54:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:07.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:07.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:03' 2026-02-22T19:54:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:54:03' 2026-02-22T19:54:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:03' 2026-02-22T19:54:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:54:03cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:54:04' 2026-02-22T19:54:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:03' 2026-02-22T19:54:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:07.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:54:07.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:07.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:54:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:54:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:54:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:54:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:54:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:54:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:54:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:54:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:54:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:54:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:54:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:54:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:54:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:54:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:54:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:54:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:54:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:54:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:08.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:08.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:08.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:54:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:54:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:54:08.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:54:08.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster1 mirror test 100 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T19:54:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T19:54:08.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T19:54:08.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T19:54:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.49 bytes/sec: 33 MiB/s 2026-02-22T19:54:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:54:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:54:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:54:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:54:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:54:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:54:12.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 23 2026-02-22T19:54:12.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:54:12.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:54:12.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:54:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:54:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:12.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:54:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:54:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:54:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:12.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:54:12.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:54:12.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:12.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:54:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:54:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:54:12.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:54:12.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=23 2026-02-22T19:54:12.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:54:12.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:54:12.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:54:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:54:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:54:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:54:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:54:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:54:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=23 2026-02-22T19:54:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 23 = 23 2026-02-22T19:54:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:54:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:54:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:54:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:09 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790047,"replay_state":"idle"} 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:08 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 23 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.e53c02cb-2f36-4866-b701-be28891cee85 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:54:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:54:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:54:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:54:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:54:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:54:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:09' 2026-02-22T19:54:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:54:09' 2026-02-22T19:54:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:09' 2026-02-22T19:54:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:54:09cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790047,"replay_state":"idle"}2026-02-22 19:54:08' 2026-02-22T19:54:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:14.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:09' 2026-02-22T19:54:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:54:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:54:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:54:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:54:14.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:14.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:14.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror test 2026-02-22T19:54:14.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:14.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:54:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:54:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790047,"replay_state":"idle"} 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:08 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:14' 2026-02-22T19:54:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:54:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:54:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790047,"replay_state":"idle"} 2026-02-22T19:54:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:54:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:54:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:08' 2026-02-22T19:54:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:54:08' 2026-02-22T19:54:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:08' 2026-02-22T19:54:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790047,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:54:08cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:54:14' 2026-02-22T19:54:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:08' 2026-02-22T19:54:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:54:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:54:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:54:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:54:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test 2026-02-22T19:54:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T19:54:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T19:54:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.export 2026-02-22T19:54:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:54:15.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T19:54:15.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.export 2026-02-22T19:54:15.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:54:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:54:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:54:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:15.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:54:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:15.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:15.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:54:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:54:15.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:15.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:54:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:54:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:54:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:54:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:54:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:54:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:54:16.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:16.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:16.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:54:16.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:54:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:54:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:54:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:54:16.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:16.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:16.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:16.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:54:16.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:54:16.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:16.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:54:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:54:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:54:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:54:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:54:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:54:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:54:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:54:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:54:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:54:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:18.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:54:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:54:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:54:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:54:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:54:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:54:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:54:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:54:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:19 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:18' 2026-02-22T19:54:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:54:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:54:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:19.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:54:19.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:19.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:19.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:19.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:19.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:19.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:19.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:19' 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:54:19' 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:19' 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:54:19cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:54:18' 2026-02-22T19:54:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:19' 2026-02-22T19:54:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:54:20.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:20.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:54:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:20.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:20.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:20.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:54:20.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:20.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:20.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:20.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:20.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:20.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:20.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:20.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:54:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:54:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:18 2026-02-22T19:54:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:54:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:19' 2026-02-22T19:54:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:54:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:54:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:54:21.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:21.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:21.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:18' 2026-02-22T19:54:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:54:18' 2026-02-22T19:54:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:18' 2026-02-22T19:54:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:54:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:54:19' 2026-02-22T19:54:21.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:21.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:18' 2026-02-22T19:54:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:54:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:54:21.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:54:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:54:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:54:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:54:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:54:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:54:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:54:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:54:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:54:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:54:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:54:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:54:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:54:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:54:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:54:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:54:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:54:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:54:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:54:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:54:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:54:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:54:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:54:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:25.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:54:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:54:25.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:25.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:25.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:54:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:54:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:54:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test 100 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T19:54:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T19:54:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T19:54:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T19:54:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.49 bytes/sec: 33 MiB/s 2026-02-22T19:54:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:54:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:54:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:54:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:54:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:54:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:54:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 26 2026-02-22T19:54:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:54:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:54:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:54:29.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:54:29.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:54:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:54:29.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:54:29.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:54:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:54:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:54:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:54:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:54:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:54:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=26 2026-02-22T19:54:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:54:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:54:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:54:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:54:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:54:29.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:54:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=26 2026-02-22T19:54:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 26 = 26 2026-02-22T19:54:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T19:54:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:54:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:54:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:54:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790061,"replay_state":"idle"} 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:24 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:28' 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:54:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790061,"replay_state":"idle"} 2026-02-22T19:54:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:54:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:54:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:24' 2026-02-22T19:54:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:54:24' 2026-02-22T19:54:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:24' 2026-02-22T19:54:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790061,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:54:24cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:54:28' 2026-02-22T19:54:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:30.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:24' 2026-02-22T19:54:30.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:30.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:54:30.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:54:30.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:54:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:31.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:31.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:28 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790061,"replay_state":"idle"} 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:24 2026-02-22T19:54:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:54:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 26 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.a49f044b-bbb3-4ced-ae9f-5a4a6c74720a (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:54:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:54:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:54:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:31.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:54:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:54:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:54:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:31.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:31.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:31.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:28' 2026-02-22T19:54:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:54:28' 2026-02-22T19:54:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:28' 2026-02-22T19:54:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:54:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790061,"replay_state":"idle"}2026-02-22 19:54:24' 2026-02-22T19:54:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:31.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:28' 2026-02-22T19:54:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:54:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:54:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:54:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:54:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test 2026-02-22T19:54:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:31.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T19:54:31.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T19:54:31.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.export 2026-02-22T19:54:31.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:31.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:54:31.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T19:54:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.export 2026-02-22T19:54:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T19:54:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:54:31.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T19:54:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T19:54:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: failover / failback loop' 2026-02-22T19:54:32.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T19:54:32.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T19:54:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 19:54:32 TEST: failover / failback loop 2026-02-22T19:54:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 19:54:32 TEST: failover / failback loop 2026-02-22T19:54:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 1 20 2026-02-22T19:54:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:54:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:54:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:54:32.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:54:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:33.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:54:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:54:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:54:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:54:33.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:33.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:54:33.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:54:33.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:33.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:33.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:33.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:33.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:54:33.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:54:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:54:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:54:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:54:33.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:54:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:54:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:54:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:54:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:54:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:35.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:54:35.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:54:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:35.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:35.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:54:35.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:54:35.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:54:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:54:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:54:36.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:36.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:36.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:34 2026-02-22T19:54:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:54:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:33' 2026-02-22T19:54:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:54:36.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:54:36.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:36.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:54:36.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:36.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:36.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:36.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:36.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:34' 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:54:34' 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:34' 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:54:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:54:33' 2026-02-22T19:54:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:34' 2026-02-22T19:54:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:36.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:54:36.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:36.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:54:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:54:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:36.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:36.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:36.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:54:37.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:54:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:33 2026-02-22T19:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:34' 2026-02-22T19:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:54:37.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:54:37.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:54:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:33' 2026-02-22T19:54:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:54:33' 2026-02-22T19:54:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:33' 2026-02-22T19:54:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:54:33cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:54:34' 2026-02-22T19:54:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:33' 2026-02-22T19:54:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:54:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:54:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:54:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:54:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:54:38.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:54:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:54:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:54:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:54:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:54:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:54:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:54:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:54:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:54:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:54:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:54:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:54:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:54:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:54:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:54:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:54:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 29 2026-02-22T19:54:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:54:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:54:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:43.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:54:43.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:54:43.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:54:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:54:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:54:43.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:54:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:54:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:54:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:54:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:54:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:54:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:54:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=29 2026-02-22T19:54:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:54:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:54:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:54:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:54:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:54:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:54:43.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:54:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:54:44.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=29 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 29 = 29 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:54:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:54:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:54:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:54:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:44 2026-02-22T19:54:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:54:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:54:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790077,"replay_state":"idle"} 2026-02-22T19:54:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:38 2026-02-22T19:54:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:54:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 29 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.079f777b-f6ce-4b2b-8da7-baa2e3c6ff77 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:54:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:54:45.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:54:45.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:45.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:54:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:54:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:54:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:45.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:44' 2026-02-22T19:54:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:54:44' 2026-02-22T19:54:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:54:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:44' 2026-02-22T19:54:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:45.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:54:44cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790077,"replay_state":"idle"}2026-02-22 19:54:38' 2026-02-22T19:54:45.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:45.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:44' 2026-02-22T19:54:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:54:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:54:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:54:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:54:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:54:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790077,"replay_state":"idle"} 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:38 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:44' 2026-02-22T19:54:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:54:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:54:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790077,"replay_state":"idle"} 2026-02-22T19:54:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:54:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:54:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:38' 2026-02-22T19:54:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:54:38' 2026-02-22T19:54:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:46.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:54:46.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:38' 2026-02-22T19:54:46.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:46.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:46.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790077,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:54:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:54:44' 2026-02-22T19:54:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:38' 2026-02-22T19:54:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:54:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:54:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:54:46.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:54:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:54:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:54:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:46.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:54:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:47.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:47.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:47.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:54:47.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:47.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:54:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:54:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:54:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:54:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:47.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:54:47.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:54:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:54:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:54:47.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:47.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:47.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:47.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:54:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:54:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:54:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:54:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:54:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:54:49.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:54:49.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:49.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:49.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:49.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:54:49.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:49.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:49.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:54:49.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:49.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:54:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:54:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:54:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:54:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:54:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:54:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:54:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:54:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:54:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:54:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:54:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:54:50.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:50.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:49 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:48' 2026-02-22T19:54:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:54:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:54:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:54:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:49' 2026-02-22T19:54:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:54:49' 2026-02-22T19:54:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:49' 2026-02-22T19:54:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:50.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:54:49cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 19:54:48' 2026-02-22T19:54:50.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:50.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:50.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:49' 2026-02-22T19:54:50.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:54:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:54:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:54:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:54:52.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:54:52.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:48 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:49' 2026-02-22T19:54:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:54:52.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:54:52.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:54:52.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:54:52.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:54:52.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:52.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:54:52.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:52.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:54:52.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:54:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:54:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:54:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:48' 2026-02-22T19:54:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:54:48' 2026-02-22T19:54:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:54:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:48' 2026-02-22T19:54:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:54:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:54:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:54:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted44434452mirror.0trial0982026-02-22 19:54:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:54:49' 2026-02-22T19:54:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:54:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:54:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:48' 2026-02-22T19:54:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:54:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:54:52.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:54:52.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:54:52.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:54:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:54:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:54:52.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:54:53.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:54:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:54:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:54:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:54:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:54:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:54:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:54:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:54:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:54:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:54:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:54:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:54:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:54.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:54:54.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:54:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:54:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:54:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:54:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:54:54.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:54.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:54.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:54.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:54:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:54:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:54:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:54:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:54:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:54:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:54:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:54:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:54:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:54:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:54:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:54:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:54:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:54:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:54:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:54:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:54:56.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:54:56.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:54:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:54:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:54:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:54:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:54:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:54:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:54:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:54:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:54:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:54:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:54:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:54:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:54:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:54:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:54:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:54:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:54:56.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:54:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:54:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:54:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:54:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:54:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:54:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:54:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:54:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:54:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:54:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:54:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:54:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:55:00.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 32 2026-02-22T19:55:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:55:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:55:00.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:00.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:00.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:00.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:55:00.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:55:00.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:55:00.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:00.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:55:00.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:55:00.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:55:00.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:55:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:55:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:55:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=32 2026-02-22T19:55:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:55:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:55:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:55:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:55:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:55:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:55:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=32 2026-02-22T19:55:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 32 = 32 2026-02-22T19:55:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:55:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:01.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:55:01.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:55:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:55:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:58 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790092,"replay_state":"idle"} 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:54 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 32 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.4fb75d6f-55fd-487e-b913-7ec61631a680 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:55:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:55:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:55:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:55:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:55:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:55:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:01.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:01.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:01.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:58' 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:54:58' 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:58' 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:01.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:54:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790092,"replay_state":"idle"}2026-02-22 19:54:54' 2026-02-22T19:55:01.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:01.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:01.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:58' 2026-02-22T19:55:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:55:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:55:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:55:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:55:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:55:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:55:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790092,"replay_state":"idle"} 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:54 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:54:58' 2026-02-22T19:55:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:55:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:55:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790092,"replay_state":"idle"} 2026-02-22T19:55:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:55:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:55:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:54' 2026-02-22T19:55:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:54:54' 2026-02-22T19:55:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:54:54' 2026-02-22T19:55:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790092,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:54:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:54:58' 2026-02-22T19:55:02.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:02.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:54:54' 2026-02-22T19:55:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:55:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:55:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:55:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:55:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:55:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:55:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:55:03.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:55:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:55:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:55:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:55:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:55:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:55:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:55:04.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:55:04.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:04.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:55:04.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:55:04.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:55:04.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:04.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:55:06.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:55:06.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:06.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:06.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:06.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:55:06.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:06.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:06.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:55:06.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:06.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:55:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:55:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:55:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:55:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:55:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:55:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:55:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:55:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:55:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:55:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:55:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:55:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:55:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:55:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:55:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:55:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:04 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:03' 2026-02-22T19:55:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:55:07.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:55:07.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:55:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:07.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:07.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:07.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:07.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:07.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:04' 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:55:04' 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:04' 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:07.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 19:55:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:55:03' 2026-02-22T19:55:07.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:07.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:07.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:04' 2026-02-22T19:55:07.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:55:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:55:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:55:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:55:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:08 2026-02-22T19:55:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:55:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:55:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:04' 2026-02-22T19:55:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:55:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:55:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:08.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:55:08.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:08.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:08.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:08.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:08.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:08.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:08' 2026-02-22T19:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:55:08' 2026-02-22T19:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:08' 2026-02-22T19:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:55:08cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 19:55:04' 2026-02-22T19:55:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:08' 2026-02-22T19:55:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:08.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:55:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:55:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:55:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:55:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:55:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:55:09.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:55:09.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:55:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:55:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:55:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:55:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:55:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:55:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:55:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:55:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:55:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:55:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:55:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:55:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:55:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:55:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:55:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:55:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:55:12.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:55:12.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:12.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:12.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:12.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:55:12.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:12.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:12.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:55:12.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:12.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:55:12.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:12.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:55:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:55:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:55:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:55:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:55:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:12.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:12.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:55:12.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:55:12.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:12.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:12.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:55:12.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:12.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:12.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:12.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:55:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:55:12.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:55:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T19:55:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:55:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:55:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:16.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:55:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:55:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:55:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:55:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:16.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:55:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:55:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:55:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:55:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:16.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:16.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:55:16.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:55:16.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:16.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:55:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:55:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 35 2026-02-22T19:55:20.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:55:20.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:55:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:55:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:55:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:55:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:55:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:55:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:55:20.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:20.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:20.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:20.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:20.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:20.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:20.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:55:20.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:55:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:55:20.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=35 2026-02-22T19:55:20.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:55:20.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:55:20.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:55:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:55:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:55:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:55:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=35 2026-02-22T19:55:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 35 = 35 2026-02-22T19:55:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:55:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:55:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:55:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:55:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:19 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790108,"remote_snapshot_timestamp":1771790108,"replay_state":"idle"} 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:18 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 35 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.c3d9a44d-67ec-4421-9e98-c030e118b346 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:55:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:55:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:55:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:55:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:22.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:22.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:19' 2026-02-22T19:55:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:55:19' 2026-02-22T19:55:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:19' 2026-02-22T19:55:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:55:19cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790108,"remote_snapshot_timestamp":1771790108,"replay_state":"idle"}2026-02-22 19:55:18' 2026-02-22T19:55:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:19' 2026-02-22T19:55:22.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:55:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:55:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:55:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:55:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:55:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:55:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:55:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:55:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790108,"remote_snapshot_timestamp":1771790108,"replay_state":"idle"} 2026-02-22T19:55:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:18 2026-02-22T19:55:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:55:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:55:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:55:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:19' 2026-02-22T19:55:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:55:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:55:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790108,"remote_snapshot_timestamp":1771790108,"replay_state":"idle"} 2026-02-22T19:55:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:55:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:55:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:18' 2026-02-22T19:55:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:55:18' 2026-02-22T19:55:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:18' 2026-02-22T19:55:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:23.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:23.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790108,"remote_snapshot_timestamp":1771790108,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:55:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:55:19' 2026-02-22T19:55:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:18' 2026-02-22T19:55:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:55:23.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:55:23.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:55:23.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:55:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:55:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:55:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:55:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:55:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:55:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:55:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:55:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:55:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:55:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:55:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:55:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:24.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:55:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:55:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:55:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:55:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:55:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:55:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:55:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:55:24.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:24.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:55:24.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:55:24.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:55:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:55:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:55:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:55:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:55:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:55:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:55:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:55:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:26.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:55:26.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:55:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:55:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:55:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:55:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:55:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:55:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:55:27.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:24 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:55:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:24' 2026-02-22T19:55:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:55:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:55:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:27.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:55:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:24' 2026-02-22T19:55:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:55:24' 2026-02-22T19:55:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:24' 2026-02-22T19:55:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:55:24cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:55:28' 2026-02-22T19:55:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:24' 2026-02-22T19:55:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:55:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:55:28.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:55:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:55:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:55:29.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:29.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:29.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:29.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:29.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:29.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:28 2026-02-22T19:55:29.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:29.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:55:29.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:29.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:29.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:24' 2026-02-22T19:55:29.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:55:29.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:55:29.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:29.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:55:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:28' 2026-02-22T19:55:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:55:28' 2026-02-22T19:55:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:28' 2026-02-22T19:55:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:55:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:55:24' 2026-02-22T19:55:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:28' 2026-02-22T19:55:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:55:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:55:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:29.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:29.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:29.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:55:29.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:29.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:29.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:29.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:55:29.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:55:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:55:31.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:55:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:55:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:55:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:55:31.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:31.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:55:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:55:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:55:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:55:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:55:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:55:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:55:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:55:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:32.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:55:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:32.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:55:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:55:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:55:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:55:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:55:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:55:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:34.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:55:34.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:55:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:55:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:55:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:34.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:55:34.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:55:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:55:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:55:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:55:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:55:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:55:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:34.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:55:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:55:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:55:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:55:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:55:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:55:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:55:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:55:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:55:38.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 38 2026-02-22T19:55:38.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:55:38.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:55:38.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:38.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:38.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:38.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:55:38.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:55:38.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:55:38.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:55:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:55:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:55:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:55:38.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:55:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:55:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=38 2026-02-22T19:55:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:55:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:55:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:55:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:55:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:55:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:55:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=38 2026-02-22T19:55:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 38 = 38 2026-02-22T19:55:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:55:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:39.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:55:39.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:39.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:39.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:38 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790130,"replay_state":"idle"} 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:34 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 38 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.024d5384-1761-4d8c-90af-7968f21d6709 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:55:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:55:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:55:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:55:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:55:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:55:39.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:39.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:38' 2026-02-22T19:55:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:55:38' 2026-02-22T19:55:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:38' 2026-02-22T19:55:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:55:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790130,"replay_state":"idle"}2026-02-22 19:55:34' 2026-02-22T19:55:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:38' 2026-02-22T19:55:39.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:55:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:55:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:55:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:55:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:55:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:40.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:55:40.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:55:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:55:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790130,"replay_state":"idle"} 2026-02-22T19:55:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:34 2026-02-22T19:55:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:55:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:55:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:55:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:38' 2026-02-22T19:55:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:55:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:55:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790130,"replay_state":"idle"} 2026-02-22T19:55:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:55:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:55:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:34' 2026-02-22T19:55:40.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:55:34' 2026-02-22T19:55:40.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:40.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:40.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:40.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:40.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:34' 2026-02-22T19:55:40.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:40.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:40.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790130,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:55:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:55:38' 2026-02-22T19:55:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:34' 2026-02-22T19:55:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:55:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:55:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:55:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:55:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:55:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:55:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:55:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:55:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:55:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:55:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:55:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:55:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:55:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:55:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:55:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:55:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:55:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:55:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:55:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:55:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:55:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:55:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:55:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:55:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:55:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:55:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:55:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:55:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:55:44.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:55:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:55:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:55:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:44.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:55:44.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:55:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:55:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:55:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:44 2026-02-22T19:55:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:55:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:43' 2026-02-22T19:55:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:55:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:55:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:55:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:44' 2026-02-22T19:55:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:55:44' 2026-02-22T19:55:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:44' 2026-02-22T19:55:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:45.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:55:44cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:55:43' 2026-02-22T19:55:45.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:45.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:45.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:44' 2026-02-22T19:55:45.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:55:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:55:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:55:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:55:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:43 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:44' 2026-02-22T19:55:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:55:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:55:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:55:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:46.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:46.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:46.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:43' 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:55:43' 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:43' 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:55:43cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:55:44' 2026-02-22T19:55:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:46.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:43' 2026-02-22T19:55:46.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:55:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:55:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:46.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:46.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:55:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:55:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:55:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:55:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:55:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:55:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:55:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:55:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:55:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:55:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:55:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:55:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:55:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:55:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:55:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:55:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:55:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:55:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:55:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:55:48.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:48.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:48.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:48.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:48.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:48.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:55:48.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:55:48.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:55:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:55:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:55:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:55:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:55:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:55:52.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 41 2026-02-22T19:55:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:55:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:55:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:55:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:55:52.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:55:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:52.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:55:52.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:55:52.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:55:52.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:55:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:55:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:55:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:55:52.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=41 2026-02-22T19:55:52.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:55:52.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:55:52.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:55:52.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:52.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:52.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:52.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:52.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:55:52.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:55:52.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:55:53.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:55:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=41 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 41 = 41 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:55:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:54.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:54.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:54.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:55:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:49 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790146,"replay_state":"idle"} 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:48 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 41 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.d83dd0ae-f590-429b-b540-44da181f6a8c (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:55:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:55:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:55:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:55:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:55:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:55:54.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:54.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:54.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:54.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:49' 2026-02-22T19:55:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:55:49' 2026-02-22T19:55:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:55:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:49' 2026-02-22T19:55:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:54.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:55:49cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790146,"replay_state":"idle"}2026-02-22 19:55:48' 2026-02-22T19:55:54.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:54.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:54.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:49' 2026-02-22T19:55:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:54.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:55:54.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:55:54.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:55:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:55:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:55:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:55:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:55:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790146,"replay_state":"idle"} 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:48 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:54' 2026-02-22T19:55:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:55:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:55:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790146,"replay_state":"idle"} 2026-02-22T19:55:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:55:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:55:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:55.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:55.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:55.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:55.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:48' 2026-02-22T19:55:55.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:55:48' 2026-02-22T19:55:55.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:55.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:55.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:55.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:55:55.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:48' 2026-02-22T19:55:55.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:55.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:55.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790146,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:55:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:55:54' 2026-02-22T19:55:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:48' 2026-02-22T19:55:55.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:55:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:55:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:55:55.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:55:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:55:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:55.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:55.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:55.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:55:55.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:55:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:55:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:55:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:56.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:56.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:55:56.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:56.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:55:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:55:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:55:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:55:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:56.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:56.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:55:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:55:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:55:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:56.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:56.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:56.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:55:56.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:55:56.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:56.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:55:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:55:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:55:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:55:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:55:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:55:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:55:58.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:58.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:55:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:55:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:55:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:55:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:55:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:55:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:55:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:55:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:55:58.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:55:58.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:55:58.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:55:58.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:55:58.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:55:58.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:55:58.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:55:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:55:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:55:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:55:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:55:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:55:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:55:58.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:55:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:55:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:55:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:55:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:55:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:55:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:55:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:55:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:59 2026-02-22T19:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:58' 2026-02-22T19:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:55:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:55:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:55:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:55:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:55:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:55:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:55:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:55:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:55:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:55:59.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:59' 2026-02-22T19:55:59.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:55:59' 2026-02-22T19:55:59.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:55:59.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:59.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:59.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:59.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:59' 2026-02-22T19:55:59.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:55:59.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:55:59.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:55:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:55:59cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:55:58' 2026-02-22T19:55:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:55:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:55:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:59' 2026-02-22T19:55:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:55:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:55:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:55:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:55:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:55:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:55:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:55:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:55:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:55:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:55:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:55:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:55:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:55:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:55:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:56:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:58 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:55:59' 2026-02-22T19:56:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:56:01.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:56:01.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:01.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:56:01.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:01.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:01.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:01.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:01.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:58' 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:55:58' 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:55:58' 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:55:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:55:59' 2026-02-22T19:56:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:01.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:55:58' 2026-02-22T19:56:01.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:56:01.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:01.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:56:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:56:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:56:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:56:02.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:56:02.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:56:02.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:56:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:56:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:56:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:56:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:56:03.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:03.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:03.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:03.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:03.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:56:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:56:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:56:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:56:05.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:56:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:56:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:56:05.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:05.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:05.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:05.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:05.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:05.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:56:05.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:56:05.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:05.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:56:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:56:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:56:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:56:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 44 2026-02-22T19:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:56:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:56:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:56:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:56:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:56:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:56:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:56:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:56:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:56:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:56:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:56:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:56:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:56:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=44 2026-02-22T19:56:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:56:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:56:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:56:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:56:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:56:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:56:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:56:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:56:09.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=44 2026-02-22T19:56:09.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 44 = 44 2026-02-22T19:56:09.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:09.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:56:09.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:09.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:09.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:09.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:56:09.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:09.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:09.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:09.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:56:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:56:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:56:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:08 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790161,"replay_state":"idle"} 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:04 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 44 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.be8ba85a-8f60-4c70-b29a-d4e1d5d3a7d6 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:56:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:56:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:56:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:56:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:56:10.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:10.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:56:10.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:10.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:10.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:10.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:08' 2026-02-22T19:56:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:56:08' 2026-02-22T19:56:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:56:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:08' 2026-02-22T19:56:10.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:10.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:10.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:56:08cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790161,"replay_state":"idle"}2026-02-22 19:56:04' 2026-02-22T19:56:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:08' 2026-02-22T19:56:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:10.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:56:10.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:56:10.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:56:10.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:56:10.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:56:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:56:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:56:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790161,"replay_state":"idle"} 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:04 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:08' 2026-02-22T19:56:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:56:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:56:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790161,"replay_state":"idle"} 2026-02-22T19:56:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:56:12.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:12.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:56:12.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:04' 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:56:04' 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:04' 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790161,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:56:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:56:08' 2026-02-22T19:56:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:12.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:04' 2026-02-22T19:56:12.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:56:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:56:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:56:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:56:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:56:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:56:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:56:13.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:56:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:56:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:56:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:56:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:56:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:56:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:56:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:56:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:56:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:56:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:56:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:56:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:56:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:56:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:56:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:56:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:56:14.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:56:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:56:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:56:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:56:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:56:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:56:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:56:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:56:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:56:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:56:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:56:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:56:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:56:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:56:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:56:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:56:16.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:16.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:16.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:16.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:16.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:56:16.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:56:16.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:16.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:56:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:14 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:13' 2026-02-22T19:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:14' 2026-02-22T19:56:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:56:14' 2026-02-22T19:56:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:14' 2026-02-22T19:56:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 19:56:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:56:13' 2026-02-22T19:56:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:17.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:14' 2026-02-22T19:56:17.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:56:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:56:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:18.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:56:18.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:18.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:18.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:18.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:18.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:18.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:18.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:18.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:18.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:56:18.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:56:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:18 2026-02-22T19:56:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:56:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:56:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:14' 2026-02-22T19:56:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:56:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:56:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:18.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:56:18.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:18.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:18.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:18' 2026-02-22T19:56:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:56:18' 2026-02-22T19:56:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:18' 2026-02-22T19:56:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:18.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:56:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 19:56:14' 2026-02-22T19:56:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:18' 2026-02-22T19:56:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:18.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:56:18.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:18.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:56:18.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:56:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:56:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:56:19.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:56:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:56:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:56:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:56:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:56:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:20.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:56:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:56:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:56:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:56:20.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:20.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:56:20.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:56:20.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:20.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:20.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:20.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:56:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:56:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:56:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:56:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:56:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:56:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:20.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:56:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:56:20.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:56:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:56:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:56:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:56:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:56:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:56:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:56:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:22.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:22.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:56:22.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:56:22.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:56:22.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:22.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:56:22.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:56:22.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:56:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:56:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:56:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:56:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:56:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:56:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T19:56:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:56:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:56:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:56:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:56:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:56:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:56:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:56:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:56:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:56:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:56:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:56:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:56:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:26.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:26.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:26.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:26.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:56:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:56:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:56:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:56:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:56:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:56:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:56:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:56:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:56:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:56:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:56:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 47 2026-02-22T19:56:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:56:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:56:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:56:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:56:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:56:30.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:56:30.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:56:30.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:56:30.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:56:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:56:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:56:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:56:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:56:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:56:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:56:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=47 2026-02-22T19:56:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:56:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:56:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:56:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:56:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:56:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:56:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:56:31.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=47 2026-02-22T19:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 47 = 47 2026-02-22T19:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:31.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:56:31.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:31.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:31.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:31.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:56:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:29 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790178,"replay_state":"idle"} 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:23 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:56:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 47 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.b3a77dca-458d-41af-a946-96cb87bed240 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:56:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:56:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:56:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:56:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:56:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:56:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:29' 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:56:29' 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:29' 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:56:29cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790178,"replay_state":"idle"}2026-02-22 19:56:23' 2026-02-22T19:56:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:32.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:29' 2026-02-22T19:56:32.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:32.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:56:32.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:56:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:56:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:56:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:56:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:56:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:56:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790178,"replay_state":"idle"} 2026-02-22T19:56:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:23 2026-02-22T19:56:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:56:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:56:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:56:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:29' 2026-02-22T19:56:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:56:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:56:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:33.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790178,"replay_state":"idle"} 2026-02-22T19:56:33.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:33.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:56:33.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:33.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:56:33.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:33.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:33.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:33.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:23' 2026-02-22T19:56:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:56:23' 2026-02-22T19:56:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:56:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:23' 2026-02-22T19:56:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790178,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:56:23cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:56:29' 2026-02-22T19:56:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:23' 2026-02-22T19:56:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:33.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:56:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:56:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:56:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:56:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:56:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:56:33.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:56:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:56:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:56:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:56:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:56:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:56:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:34.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:56:34.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:34.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:34.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:34.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:56:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:56:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:56:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:56:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:56:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:56:36.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:56:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:56:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:56:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:56:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:56:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:56:36.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:36.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:36.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:36.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:56:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:56:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:56:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:56:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:56:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:56:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:56:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:56:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:56:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:36.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:56:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:56:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:56:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:34 2026-02-22T19:56:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:56:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:56:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:35' 2026-02-22T19:56:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:56:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:56:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:56:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:37.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:37.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:37.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:37.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:34' 2026-02-22T19:56:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:56:34' 2026-02-22T19:56:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:34' 2026-02-22T19:56:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:37.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:56:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 19:56:35' 2026-02-22T19:56:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:34' 2026-02-22T19:56:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:38.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:56:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:56:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:56:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:38 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:34' 2026-02-22T19:56:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:56:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:56:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:56:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:38' 2026-02-22T19:56:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:56:38' 2026-02-22T19:56:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:38' 2026-02-22T19:56:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:56:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:56:34' 2026-02-22T19:56:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:38' 2026-02-22T19:56:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:56:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:56:39.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:56:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:56:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:56:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:56:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:56:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:56:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:56:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:56:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:56:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:56:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:56:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:56:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:56:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:56:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:56:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:56:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:56:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:56:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:56:45.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 50 2026-02-22T19:56:45.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:56:45.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:56:45.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:56:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:56:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:56:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:56:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:56:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:56:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:56:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:56:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:56:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:56:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:56:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:56:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:56:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=50 2026-02-22T19:56:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:56:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:56:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:56:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:56:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:56:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:56:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:56:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:56:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=50 2026-02-22T19:56:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 50 = 50 2026-02-22T19:56:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:56:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:56:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:45.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:56:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:43 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790199,"replay_state":"idle"} 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:39 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 50 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.385d668e-5fc8-4659-b587-1dabb056a38a (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:56:46.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:56:46.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:56:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:56:46.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:43' 2026-02-22T19:56:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:56:43' 2026-02-22T19:56:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:56:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:43' 2026-02-22T19:56:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:56:43cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790199,"replay_state":"idle"}2026-02-22 19:56:39' 2026-02-22T19:56:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:46.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:46.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:43' 2026-02-22T19:56:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:56:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:56:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:56:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:56:46.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:46.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:46.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:56:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:56:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:47.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:56:47.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:47.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:47.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:56:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:56:47.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:47.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:47.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:56:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790199,"replay_state":"idle"} 2026-02-22T19:56:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:39 2026-02-22T19:56:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:56:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:56:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:56:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:43' 2026-02-22T19:56:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:56:47.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:56:47.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:47.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790199,"replay_state":"idle"} 2026-02-22T19:56:47.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:47.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:56:47.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:47.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:56:47.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:39' 2026-02-22T19:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:56:39' 2026-02-22T19:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:39' 2026-02-22T19:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:47.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:47.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790199,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:56:39cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:56:43' 2026-02-22T19:56:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:39' 2026-02-22T19:56:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:56:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:56:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:56:47.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:56:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:56:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:56:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:56:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:56:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:56:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:56:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:56:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:56:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:49.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:56:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:56:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:56:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:56:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:56:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:56:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:56:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:56:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:56:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:56:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:56:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:56:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:49.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:49.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:49.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:49.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:49.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:56:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:56:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:56:49.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:56:49.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:56:49.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:49.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:56:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:56:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:56:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:56:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:56:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:56:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:56:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:51.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:56:51.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:56:51.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:51.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:51.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:56:51.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:56:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:56:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:51.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:56:51.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:51.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:56:51.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:51.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:51.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:51.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:51.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:51.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:51.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:51.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:56:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:49 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:56:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:56:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:48' 2026-02-22T19:56:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:56:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:56:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:56:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:49' 2026-02-22T19:56:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:56:49' 2026-02-22T19:56:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:49' 2026-02-22T19:56:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 19:56:49cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:56:48' 2026-02-22T19:56:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:49' 2026-02-22T19:56:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:52.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:56:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:56:52.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:56:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:56:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:56:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:56:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:56:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:56:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:56:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:56:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:56:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:53 2026-02-22T19:56:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:56:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:56:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:56:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:56:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:49' 2026-02-22T19:56:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:56:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:56:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:56:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:56:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:56:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:56:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:56:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:56:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:56:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:56:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:53' 2026-02-22T19:56:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:56:53' 2026-02-22T19:56:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:56:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:53' 2026-02-22T19:56:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:56:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:56:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:56:53.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:56:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 19:56:49' 2026-02-22T19:56:53.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:56:53.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:56:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:53' 2026-02-22T19:56:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:56:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:56:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:56:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:56:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:56:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:56:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:56:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:56:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:56:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:56:54.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:56:54.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:56:54.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:54.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:54.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:54.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:56:54.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:54.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:54.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:54.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:56:54.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:56:54.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:54.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:56:55.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:56:55.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:55.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:55.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:55.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:56:55.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:55.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:55.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:56:55.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:56:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:56:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:56:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:56:55.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:55.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:55.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:56:55.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:56:55.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:55.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:55.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:56:55.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:56:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:56:55.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:55.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:55.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:55.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:56:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:56:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:56:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:56:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:56:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:56:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:56:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:56:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:56:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:56:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:56:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:56:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:56:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:56:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:56:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:56:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:56:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:56:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:56:57.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:56:57.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:56:57.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:56:57.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:56:57.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:56:57.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:56:57.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:56:57.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:56:57.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:56:57.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:56:57.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:56:57.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:56:57.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:56:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:56:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:56:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:56:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:56:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:56:57.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:56:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:56:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:56:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:56:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:56:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:56:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:56:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:56:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:56:57.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T19:57:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:57:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:57:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:57:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:57:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:57:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:57:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:57:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:01.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:01.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:57:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:57:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:57:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:57:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:57:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:57:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:57:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:57:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:57:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:57:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:57:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:57:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 53 2026-02-22T19:57:05.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:57:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:57:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:57:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:57:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:57:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:57:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:57:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:57:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:57:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:57:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:57:05.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=53 2026-02-22T19:57:05.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:57:05.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:57:05.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:57:05.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:05.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:05.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:57:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:57:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:57:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=53 2026-02-22T19:57:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 53 = 53 2026-02-22T19:57:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:57:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:57:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:57:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:57:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:04 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790213,"replay_state":"idle"} 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:58 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 53 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.abde6f73-cde6-41c0-984a-bcf5aaeed751 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:57:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:57:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:57:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:57:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:57:07.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:07.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:57:07.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:04' 2026-02-22T19:57:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:57:04' 2026-02-22T19:57:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:04' 2026-02-22T19:57:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:57:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790213,"replay_state":"idle"}2026-02-22 19:56:58' 2026-02-22T19:57:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:07.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:04' 2026-02-22T19:57:07.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:57:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:57:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:57:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:57:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:57:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:08.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:57:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:57:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790213,"replay_state":"idle"} 2026-02-22T19:57:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:56:58 2026-02-22T19:57:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:57:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:57:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:57:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:04' 2026-02-22T19:57:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:57:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:57:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790213,"replay_state":"idle"} 2026-02-22T19:57:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:57:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:57:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:08.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:08.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:58' 2026-02-22T19:57:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:56:58' 2026-02-22T19:57:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:56:58' 2026-02-22T19:57:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790213,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:56:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:57:04' 2026-02-22T19:57:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:56:58' 2026-02-22T19:57:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:57:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:57:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:57:08.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:57:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:57:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:57:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:57:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:57:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:57:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:57:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:57:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:57:09.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:09.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:57:09.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:09.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:57:09.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:57:09.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:09.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:09.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:09.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:57:09.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:09.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:09.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:09.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:09.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:09.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:57:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:57:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:57:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:57:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:57:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:57:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:57:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:57:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:57:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:11.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:57:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:11.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:57:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:57:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:57:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:11.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:11.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:57:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:57:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:57:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:57:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:57:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:57:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:09 2026-02-22T19:57:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:57:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:57:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:10' 2026-02-22T19:57:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:57:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:57:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:57:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:12.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:12.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:12.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:09' 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:57:09' 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:09' 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:57:09cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 19:57:10' 2026-02-22T19:57:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:12.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:09' 2026-02-22T19:57:12.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:57:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:14.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:57:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:57:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:57:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:13 2026-02-22T19:57:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:57:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:57:14.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:09' 2026-02-22T19:57:14.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:57:14.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:57:14.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:57:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:14.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:14.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:13' 2026-02-22T19:57:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:57:13' 2026-02-22T19:57:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:13' 2026-02-22T19:57:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:57:13cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:57:09' 2026-02-22T19:57:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:14.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:13' 2026-02-22T19:57:14.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:14.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:57:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:57:14.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:57:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:57:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:57:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:57:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:57:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:57:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:57:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:57:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:57:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:57:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:57:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:57:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:57:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:57:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:57:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:57:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:57:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:57:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:57:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:57:16.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:16.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:16.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:16.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:57:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:57:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:57:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:57:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:57:20.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 56 2026-02-22T19:57:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:57:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:57:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:57:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:57:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:57:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:20.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:57:20.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:57:20.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:57:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:57:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:57:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=56 2026-02-22T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:57:20.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:57:20.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:20.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:20.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:20.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:20.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:20.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:57:20.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:57:20.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:57:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=56 2026-02-22T19:57:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 56 = 56 2026-02-22T19:57:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:57:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:20.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:20.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:20.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:20.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:20.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:20.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:21.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:57:21.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:57:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:57:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:57:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:18 2026-02-22T19:57:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:57:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:57:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790234,"replay_state":"idle"} 2026-02-22T19:57:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:14 2026-02-22T19:57:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:57:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 56 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.cace4c05-c26a-4e0c-a03a-ea872ce05da7 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:57:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:57:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:57:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:57:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:57:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:57:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:18' 2026-02-22T19:57:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:57:18' 2026-02-22T19:57:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:18' 2026-02-22T19:57:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:57:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790234,"replay_state":"idle"}2026-02-22 19:57:14' 2026-02-22T19:57:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:18' 2026-02-22T19:57:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:57:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:57:21.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:57:21.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:57:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:57:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:21.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:21.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:21.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:21.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:22.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:57:22.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:57:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:57:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790234,"replay_state":"idle"} 2026-02-22T19:57:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:14 2026-02-22T19:57:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:57:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:57:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:57:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:18' 2026-02-22T19:57:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:57:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:57:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790234,"replay_state":"idle"} 2026-02-22T19:57:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:57:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:57:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:22.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:22.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:14' 2026-02-22T19:57:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:57:14' 2026-02-22T19:57:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:14' 2026-02-22T19:57:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790234,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:57:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:57:18' 2026-02-22T19:57:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:14' 2026-02-22T19:57:22.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:57:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:57:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:57:22.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:57:22.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:57:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:57:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:57:23.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:57:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:57:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:57:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:57:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:57:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:57:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:57:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:57:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:57:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:57:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:57:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:57:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:57:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:57:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:57:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:57:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:57:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:57:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:57:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:57:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:57:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:57:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:57:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:57:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:57:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:57:24.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:57:24.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:57:24.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:57:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:57:26.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:57:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:57:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:26.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:57:26.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:26.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:26.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:26.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:26.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:26.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:57:26.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:57:26.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:57:27.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:24 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:23' 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:57:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:57:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:57:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:24' 2026-02-22T19:57:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:57:24' 2026-02-22T19:57:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:24' 2026-02-22T19:57:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:27.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:27.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 19:57:24cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:57:23' 2026-02-22T19:57:27.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:27.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:27.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:24' 2026-02-22T19:57:27.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:57:27.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:27.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:57:27.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:57:28.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:28.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:28.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:28.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:57:28.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:28 2026-02-22T19:57:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:57:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:57:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:24' 2026-02-22T19:57:28.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:57:28.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:57:28.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:28.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:57:28.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:28.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:28.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:28.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:28' 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:57:28' 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:28' 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:28.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:57:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 19:57:24' 2026-02-22T19:57:28.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:28.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:28' 2026-02-22T19:57:28.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:57:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:57:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:57:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:57:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:57:29.743 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:57:29.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:57:29.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:29.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:57:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:57:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:57:30.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:57:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:30.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:57:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:57:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:57:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:57:32.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:57:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:57:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:57:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:57:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:57:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:57:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:32.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:57:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:32.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:57:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:57:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:32.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:57:32.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:32.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T19:57:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:57:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:57:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:57:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:57:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:57:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:57:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:57:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:57:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:57:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:57:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:57:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:57:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:57:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:57:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:57:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:57:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:57:40.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 59 2026-02-22T19:57:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:57:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:57:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:57:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:57:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:57:40.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:57:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:57:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:57:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:57:40.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:57:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:57:40.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=59 2026-02-22T19:57:40.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:57:40.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:57:40.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:57:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:57:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:57:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:57:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=59 2026-02-22T19:57:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 59 = 59 2026-02-22T19:57:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:57:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:57:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:57:42.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:42.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:42.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:57:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:57:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:39 2026-02-22T19:57:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:57:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:57:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790248,"replay_state":"idle"} 2026-02-22T19:57:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:33 2026-02-22T19:57:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:57:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 59 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.68fbfeb8-be4c-40df-b3a9-bebfe14dc4fa (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:57:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:57:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:57:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:57:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:57:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:57:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:39' 2026-02-22T19:57:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:57:39' 2026-02-22T19:57:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:39' 2026-02-22T19:57:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:42.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:57:39cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790248,"replay_state":"idle"}2026-02-22 19:57:33' 2026-02-22T19:57:42.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:42.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:39' 2026-02-22T19:57:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:42.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:57:42.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:57:42.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:57:42.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:57:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:57:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:57:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:57:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:57:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790248,"replay_state":"idle"} 2026-02-22T19:57:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:33 2026-02-22T19:57:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:57:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:57:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:57:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:39' 2026-02-22T19:57:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:57:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:57:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790248,"replay_state":"idle"} 2026-02-22T19:57:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:57:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:57:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:33' 2026-02-22T19:57:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:57:33' 2026-02-22T19:57:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:33' 2026-02-22T19:57:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790248,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:57:33cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:57:39' 2026-02-22T19:57:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:43.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:33' 2026-02-22T19:57:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:43.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:57:43.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:57:43.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:57:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:57:43.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:43.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:57:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:57:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:57:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:57:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:57:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:57:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:57:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:57:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:57:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:57:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:57:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:57:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:57:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:57:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:57:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:57:44.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:44.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:57:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:57:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:57:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:57:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:57:44.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:44.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:57:46.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:57:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:57:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:57:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:57:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:57:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:57:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:57:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:57:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:57:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:57:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:57:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:57:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:57:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:57:46.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:46.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:57:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:57:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:57:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:57:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:57:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:47.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:57:47.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:47.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:47.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:47.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:47.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:47.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:47.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:47.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:47.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:57:47.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:47.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:47.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:47.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:57:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:44 2026-02-22T19:57:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:57:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:57:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:45' 2026-02-22T19:57:47.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:57:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:57:47.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:47.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:57:47.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:47.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:47.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:47.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:44' 2026-02-22T19:57:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:57:44' 2026-02-22T19:57:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:47.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:47.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:44' 2026-02-22T19:57:47.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:47.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:47.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:57:44cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 19:57:45' 2026-02-22T19:57:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:44' 2026-02-22T19:57:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:57:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:57:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:57:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:57:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:57:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:57:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:48 2026-02-22T19:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:44' 2026-02-22T19:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:48' 2026-02-22T19:57:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:57:48' 2026-02-22T19:57:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:57:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:48' 2026-02-22T19:57:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:57:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:57:44' 2026-02-22T19:57:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:48' 2026-02-22T19:57:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:49.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:57:49.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:57:49.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:57:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:57:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:57:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:57:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:57:50.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:57:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:57:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:57:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:57:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:57:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:57:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:57:51.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:51.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:57:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:57:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:57:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:57:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:57:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:57:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:57:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:57:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:51.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:51.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:57:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:57:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:57:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:57:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:57:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:57:55.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 62 2026-02-22T19:57:55.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:57:55.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:57:55.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:55.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:55.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:55.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:57:55.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:57:55.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:57:55.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:55.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:57:55.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:57:55.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:57:55.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:55.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:55.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:55.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:55.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:55.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:57:55.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:57:55.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:57:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:57:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=62 2026-02-22T19:57:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:57:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:57:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:57:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:57:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:57:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:57:55.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=62 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 62 = 62 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:57:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:53 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790272,"remote_snapshot_timestamp":1771790272,"replay_state":"idle"} 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:54 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 62 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.3fc4e2ed-4a2c-4a69-92a0-61daf446266b (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:57:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:57:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:57:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:57:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:57:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:57:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:53' 2026-02-22T19:57:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:57:53' 2026-02-22T19:57:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:57:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:53' 2026-02-22T19:57:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:57:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790272,"remote_snapshot_timestamp":1771790272,"replay_state":"idle"}2026-02-22 19:57:54' 2026-02-22T19:57:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:53' 2026-02-22T19:57:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:56.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:57:56.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:57:56.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:57:56.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:57:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:57:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:57:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790272,"remote_snapshot_timestamp":1771790272,"replay_state":"idle"} 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:54 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:53' 2026-02-22T19:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:57:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:57:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:57:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790272,"remote_snapshot_timestamp":1771790272,"replay_state":"idle"} 2026-02-22T19:57:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:57:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:57:57.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:57:57.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:57:57.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:57:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:57:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:57:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:54' 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:57:54' 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:54' 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:57:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:57:57.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790272,"remote_snapshot_timestamp":1771790272,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:57:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:57:53' 2026-02-22T19:57:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:57:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:57:57.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:54' 2026-02-22T19:57:57.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:57:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:57:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:57:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:57:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:57:58.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:58.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:57:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:57:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:57:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:57:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:57:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:57:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:57:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:57:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:57:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:57:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:57:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:57:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:57:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:57:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:57:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:57:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:57:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:57:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:57:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:57:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:57:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:57:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:57:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:57:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:58:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:58:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:58:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:58:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:58:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:58:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:58:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:58:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:58:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:58:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:58:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:58:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:58:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:58:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:58:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:58:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:58:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:58:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:59 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:58' 2026-02-22T19:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:58:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:58:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:58:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:59' 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:57:59' 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:57:59' 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:02.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 19:57:59cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:57:58' 2026-02-22T19:58:02.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:02.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:02.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:57:59' 2026-02-22T19:58:02.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:02.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:58:02.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:02.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:58:02.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:58:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:03.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:03.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:58:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:03 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:57:59' 2026-02-22T19:58:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:58:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:58:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:58:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:03' 2026-02-22T19:58:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:58:03' 2026-02-22T19:58:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:03' 2026-02-22T19:58:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:58:03cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 19:57:59' 2026-02-22T19:58:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:03' 2026-02-22T19:58:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:03.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:58:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:58:03.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:58:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:58:03.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:58:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:58:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:58:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:58:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:58:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:58:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:58:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:58:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:58:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:58:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:58:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:58:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:07.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T19:58:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:58:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:58:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:11.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:11.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:11.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:11.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:11.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:11.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:12.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:12.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:58:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:58:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:58:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 65 2026-02-22T19:58:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:58:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:58:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:58:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:58:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:58:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:58:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:58:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:58:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:58:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:58:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:58:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:58:15.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:58:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=65 2026-02-22T19:58:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:58:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:58:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:58:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:58:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:58:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:58:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:58:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:58:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=65 2026-02-22T19:58:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 65 = 65 2026-02-22T19:58:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:58:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:58:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:16.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:16.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:16.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:58:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:58:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:17.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:17.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:58:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:17.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:17.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:58:17.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:58:17.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:17.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:14 2026-02-22T19:58:17.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:58:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:58:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790283,"replay_state":"idle"} 2026-02-22T19:58:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:08 2026-02-22T19:58:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:58:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 65 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.20362cb3-07f2-49a7-92f6-3276cec22c1e (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:58:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:58:17.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:58:17.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:58:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:58:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:58:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:14' 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:58:14' 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:14' 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:58:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790283,"replay_state":"idle"}2026-02-22 19:58:08' 2026-02-22T19:58:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:14' 2026-02-22T19:58:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:17.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:58:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:58:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:58:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:58:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:18.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:58:18.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:18.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:58:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:58:18.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:18.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:18.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:58:18.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790292,"remote_snapshot_timestamp":1771790292,"replay_state":"idle"} 2026-02-22T19:58:18.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:18.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:18 2026-02-22T19:58:18.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:18.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:58:18.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:58:18.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:58:18.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:14' 2026-02-22T19:58:18.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:58:18.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:58:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790292,"remote_snapshot_timestamp":1771790292,"replay_state":"idle"} 2026-02-22T19:58:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:58:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:58:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:18.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:18.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:18' 2026-02-22T19:58:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:58:18' 2026-02-22T19:58:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:58:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:18' 2026-02-22T19:58:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790292,"remote_snapshot_timestamp":1771790292,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:58:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:58:14' 2026-02-22T19:58:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:18' 2026-02-22T19:58:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:58:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:58:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:58:18.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:58:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:58:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:58:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:58:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:58:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:58:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:58:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:58:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:58:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:58:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:58:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:58:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:19.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:58:19.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:58:19.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:58:19.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:19.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:58:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:58:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:58:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:21.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:21.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:21.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:21.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:21.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:21.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:21.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:58:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:58:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:21.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:58:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:19 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:20' 2026-02-22T19:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:58:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:19' 2026-02-22T19:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:58:19' 2026-02-22T19:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:19' 2026-02-22T19:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:58:19cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 19:58:20' 2026-02-22T19:58:23.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:23.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:19' 2026-02-22T19:58:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:58:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:58:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:23.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:23.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:23.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:58:23.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:23.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:23.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:58:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:23 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:19' 2026-02-22T19:58:24.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:58:24.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:58:24.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:24.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:58:24.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:24.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:24.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:24.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:24.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:24.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:24.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:24.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:23' 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:58:23' 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:23' 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:58:23cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:58:19' 2026-02-22T19:58:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:23' 2026-02-22T19:58:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:58:24.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:24.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:58:24.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:24.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:24.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:24.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:58:24.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:24.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:24.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:24.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:58:24.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:58:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:58:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:58:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:58:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:58:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:58:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:58:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:58:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:26.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:58:26.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:26.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:58:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:58:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:58:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:58:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:58:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:58:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:58:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:58:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:58:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:58:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:58:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:58:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:58:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 68 2026-02-22T19:58:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:58:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:58:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:58:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:58:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:58:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:58:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:58:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:58:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:58:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:58:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:58:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:58:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:58:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:58:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:58:30.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=68 2026-02-22T19:58:30.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:58:30.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:58:30.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:58:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:58:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:58:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:58:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:58:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:58:30.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=68 2026-02-22T19:58:30.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 68 = 68 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:31.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:58:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:58:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:28 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790304,"replay_state":"idle"} 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:24 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 68 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.ee374cb8-327c-4694-b293-297c4a5a6707 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:58:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:58:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:58:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:58:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:58:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:58:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:28' 2026-02-22T19:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:58:28' 2026-02-22T19:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:28' 2026-02-22T19:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:31.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:58:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790304,"replay_state":"idle"}2026-02-22 19:58:24' 2026-02-22T19:58:31.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:31.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:31.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:28' 2026-02-22T19:58:31.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:31.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:58:31.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:58:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:58:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:58:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:58:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:58:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:58:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790304,"replay_state":"idle"} 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:24 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:28' 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790304,"replay_state":"idle"} 2026-02-22T19:58:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:58:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:58:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:32.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:24' 2026-02-22T19:58:32.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:58:24' 2026-02-22T19:58:32.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:32.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:32.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:32.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:58:32.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:24' 2026-02-22T19:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:32.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790304,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:58:24cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:58:28' 2026-02-22T19:58:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:24' 2026-02-22T19:58:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:58:32.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:58:32.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:58:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:58:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:58:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:58:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:58:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:58:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:58:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:58:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:58:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:58:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:58:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:58:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:58:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:58:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:58:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:58:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:58:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:58:34.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:34.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:58:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:58:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:58:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:58:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:58:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:58:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:58:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:58:36.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:58:36.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:36.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:36.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:36.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:58:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:58:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:58:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:58:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:58:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:58:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:58:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:36.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:58:36.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:58:36.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:36.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:58:36.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:36.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:58:36.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:36.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:36.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:36.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:36.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:36.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:36.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:36.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:58:37.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:37.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:37.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:37.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:37.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:37.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:37.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:37.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:37.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:58:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:58:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:34 2026-02-22T19:58:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:58:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:58:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:58:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:33' 2026-02-22T19:58:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:58:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:58:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T19:58:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:34' 2026-02-22T19:58:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:58:34' 2026-02-22T19:58:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:34' 2026-02-22T19:58:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 19:58:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:58:33' 2026-02-22T19:58:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:34' 2026-02-22T19:58:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:58:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:58:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:58:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:58:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:58:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:38 2026-02-22T19:58:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:58:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:58:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:34' 2026-02-22T19:58:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:58:38.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:58:38.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:38.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:58:38.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:38.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:38.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:38.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:38.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:38.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:38.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:38.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:38' 2026-02-22T19:58:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:58:38' 2026-02-22T19:58:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:38' 2026-02-22T19:58:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:38.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:38.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:58:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 19:58:34' 2026-02-22T19:58:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:38.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:38' 2026-02-22T19:58:38.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:58:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:58:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:58:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:58:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:58:39.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:58:39.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:39.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:39.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:39.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:58:39.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:58:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:58:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:58:40.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:40.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:40.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:40.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:58:40.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:40.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:40.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:40.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:40.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:40.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:40.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:40.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:40.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:40.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:40.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:40.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:40.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:58:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:58:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:58:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:58:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:58:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:58:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:42.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:42.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:58:42.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:58:42.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:42.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:58:42.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:42.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:46.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:46.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:46.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:58:46.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:58:46.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:46.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:58:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 71 2026-02-22T19:58:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:58:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:58:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:58:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:58:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:58:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:58:50.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:58:50.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:58:50.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:58:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:58:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:58:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:58:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:58:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:58:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:58:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=71 2026-02-22T19:58:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:58:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:58:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:58:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:58:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:58:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:58:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:58:51.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:58:51.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=71 2026-02-22T19:58:51.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 71 = 71 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:58:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:58:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:52.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:58:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:49 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790318,"remote_snapshot_timestamp":1771790318,"replay_state":"idle"} 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:48 2026-02-22T19:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:58:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 71 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.3cc574e1-5193-4a2e-8a95-4bc654ed4e43 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:58:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:58:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:58:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:58:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:58:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:58:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:52.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:49' 2026-02-22T19:58:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:58:49' 2026-02-22T19:58:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:58:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:49' 2026-02-22T19:58:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:58:49cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790318,"remote_snapshot_timestamp":1771790318,"replay_state":"idle"}2026-02-22 19:58:48' 2026-02-22T19:58:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:49' 2026-02-22T19:58:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:58:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:58:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:58:52.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:58:52.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:52.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:52.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:58:52.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:52.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:58:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:58:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:58:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:58:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790318,"remote_snapshot_timestamp":1771790318,"replay_state":"idle"} 2026-02-22T19:58:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:48 2026-02-22T19:58:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:58:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:58:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:58:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:49' 2026-02-22T19:58:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:58:53.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:58:53.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:53.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790318,"remote_snapshot_timestamp":1771790318,"replay_state":"idle"} 2026-02-22T19:58:53.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:53.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:58:53.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:53.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:58:53.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:53.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:53.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:53.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:48' 2026-02-22T19:58:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:58:48' 2026-02-22T19:58:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:58:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:48' 2026-02-22T19:58:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790318,"remote_snapshot_timestamp":1771790318,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:58:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:58:49' 2026-02-22T19:58:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:48' 2026-02-22T19:58:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:58:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:58:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:58:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:58:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:58:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:58:53.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:58:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:58:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:54.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:54.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:58:54.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:58:54.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:58:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:58:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:58:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:58:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:58:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:58:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:58:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:58:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:58:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:58:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:58:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:58:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:58:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:58:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:58:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:58:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:58:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:58:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:58:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:58:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:58:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:58:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:58:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:58:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:58:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:58:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:58:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:58:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:58:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:58:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:58:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:58:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:58:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:58:57.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:54 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T19:58:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:55' 2026-02-22T19:58:57.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:58:57.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:58:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:58:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:57.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:57.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:57.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:57.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:54' 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:58:54' 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:54' 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:58:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 19:58:55' 2026-02-22T19:58:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:54' 2026-02-22T19:58:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:58:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:58:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:58:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:58:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:58:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:58:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:58:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:58:59.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:58:59.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:58:59.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:59.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:58:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:58:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:58 2026-02-22T19:58:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:58:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:58:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:58:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:58:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:58:54' 2026-02-22T19:58:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:58:59.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:58:59.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:58:59.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:58:59.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:58:59.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:59.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:58:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:58:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:58:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:58:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:58:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:58' 2026-02-22T19:58:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:58:58' 2026-02-22T19:58:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:58:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:58:58' 2026-02-22T19:58:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:58:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:58:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:58:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:58:59' 2026-02-22T19:58:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:58:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:58:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:58:58' 2026-02-22T19:58:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:58:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:58:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:58:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:58:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:58:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:58:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:58:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:58:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:58:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:58:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:58:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:59:01.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:59:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:59:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:59:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:59:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:59:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:59:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:59:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:59:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:59:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:59:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:59:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:59:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:02.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:59:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:02.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:59:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:59:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:59:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:59:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:59:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:59:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:59:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:59:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:59:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:59:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:59:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:59:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:59:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:59:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:59:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:59:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:59:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:59:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:59:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 74 2026-02-22T19:59:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:59:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:59:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:59:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:59:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:59:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:59:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:59:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:59:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:59:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:59:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:59:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=74 2026-02-22T19:59:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:59:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:59:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:59:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:59:08.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:59:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:59:08.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=74 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 74 = 74 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:09.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:08 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790340,"replay_state":"idle"} 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:04 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 74 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.fb8dd4d3-f830-4c04-9333-adbd219fe357 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:59:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:59:09.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:59:09.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:09.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:59:09.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:09.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:59:09.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:09.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:59:09.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:09.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:09.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:09.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:08' 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:59:08' 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:08' 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:09.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:59:08cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790340,"replay_state":"idle"}2026-02-22 19:59:04' 2026-02-22T19:59:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:08' 2026-02-22T19:59:09.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:09.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:59:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:59:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:59:09.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:59:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:10.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:59:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:59:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790340,"replay_state":"idle"} 2026-02-22T19:59:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:04 2026-02-22T19:59:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:59:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:59:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:59:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:08' 2026-02-22T19:59:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:59:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:59:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790340,"replay_state":"idle"} 2026-02-22T19:59:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:59:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:59:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:10.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:04' 2026-02-22T19:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:59:04' 2026-02-22T19:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:04' 2026-02-22T19:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:10.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790340,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:59:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:59:08' 2026-02-22T19:59:10.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:10.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:10.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:04' 2026-02-22T19:59:10.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:59:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:59:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:59:10.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:59:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:59:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:59:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:59:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:59:11.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:59:11.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:59:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:59:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:59:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:59:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:59:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:59:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:59:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:59:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:59:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:59:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:59:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:59:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:59:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:59:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:59:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:59:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:59:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:59:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:59:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:59:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:59:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:59:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:59:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:59:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:59:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:59:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:59:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:59:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:59:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:59:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:14 2026-02-22T19:59:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:59:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:13' 2026-02-22T19:59:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:59:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:59:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:59:15.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:15.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:15.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:14' 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:59:14' 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:14' 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:59:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:59:13' 2026-02-22T19:59:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:14' 2026-02-22T19:59:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:15.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:59:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:59:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:59:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:59:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:13 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:14' 2026-02-22T19:59:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:59:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:59:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:59:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:16.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:16.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:13' 2026-02-22T19:59:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:59:13' 2026-02-22T19:59:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:13' 2026-02-22T19:59:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:59:13cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:59:14' 2026-02-22T19:59:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:13' 2026-02-22T19:59:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:59:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:59:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:59:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:59:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:59:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:59:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:59:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:59:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:17.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:17.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:59:17.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:59:17.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:17.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:59:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:59:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:59:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:59:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:59:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:59:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:59:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:59:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:59:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 77 2026-02-22T19:59:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:59:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:59:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:59:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:59:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:59:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:22.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:59:22.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:59:22.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:59:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:59:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:59:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=77 2026-02-22T19:59:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:59:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:59:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:59:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=77 2026-02-22T19:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 77 = 77 2026-02-22T19:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:59:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:59:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:19 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790356,"replay_state":"idle"} 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:18 2026-02-22T19:59:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:59:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 77 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.e21b6050-1b54-4d61-877b-8f111174f45a (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:59:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:59:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:59:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:59:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:59:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:59:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:19' 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:59:19' 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:19' 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:59:19cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790356,"replay_state":"idle"}2026-02-22 19:59:18' 2026-02-22T19:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:19' 2026-02-22T19:59:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:59:24.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:59:24.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:59:24.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:25.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:59:25.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:25.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:59:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:59:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790356,"replay_state":"idle"} 2026-02-22T19:59:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:18 2026-02-22T19:59:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:59:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:59:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:59:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:24' 2026-02-22T19:59:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:59:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:59:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790356,"replay_state":"idle"} 2026-02-22T19:59:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:59:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:59:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:18' 2026-02-22T19:59:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:59:18' 2026-02-22T19:59:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:18' 2026-02-22T19:59:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:25.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790356,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:59:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:59:24' 2026-02-22T19:59:25.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:25.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:18' 2026-02-22T19:59:25.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:25.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:59:25.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:59:25.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:59:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:59:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:59:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:25.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:25.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:59:25.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:59:25.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:59:25.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:25.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:25.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:25.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:59:25.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:25.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:25.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:25.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:59:25.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:59:25.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:25.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:26.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:59:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:59:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:26.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:59:26.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:26.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:59:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:59:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:59:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:59:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:59:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:59:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:59:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:59:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:59:26.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:59:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:59:26.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:59:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:59:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:59:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:59:28.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:59:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:59:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:59:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:59:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:28.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:28.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:59:28.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:59:28.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:28.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:28.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:59:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:59:29.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:29.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:29.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:29.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:29.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:29.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:29.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:29.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:29.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:59:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:29 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:28' 2026-02-22T19:59:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:59:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:59:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:59:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:29.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:29.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:29.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:29.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:29.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:29' 2026-02-22T19:59:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:59:29' 2026-02-22T19:59:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:29' 2026-02-22T19:59:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:59:29cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:59:28' 2026-02-22T19:59:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:29' 2026-02-22T19:59:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:59:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:59:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:59:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:28 2026-02-22T19:59:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:59:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:29' 2026-02-22T19:59:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:59:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:59:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:59:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:31.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:31.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:31.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:28' 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:59:28' 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:28' 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:59:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:59:29' 2026-02-22T19:59:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:31.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:28' 2026-02-22T19:59:31.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:59:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:59:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T19:59:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:59:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T19:59:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:59:32.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T19:59:32.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:59:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:59:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:59:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:59:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:59:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:59:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:59:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T19:59:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:59:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T19:59:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:59:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:59:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:59:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:59:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:59:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:59:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:59:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:59:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:59:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:59:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:59:35.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:35.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:35.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:35.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:59:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:59:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:59:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:59:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:59:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T19:59:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:59:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T19:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T19:59:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 80 2026-02-22T19:59:39.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T19:59:39.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T19:59:39.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:39.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:39.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:39.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:59:39.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:59:39.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:59:39.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:39.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:59:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:59:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:59:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:39.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:39.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:39.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:39.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:59:39.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:59:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:59:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=80 2026-02-22T19:59:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:59:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:59:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:59:39.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:39.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:39.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:39.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:39.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:39.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:59:39.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:59:39.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:59:39.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=80 2026-02-22T19:59:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 80 = 80 2026-02-22T19:59:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T19:59:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:39.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:39.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T19:59:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:59:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:59:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:59:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:38 2026-02-22T19:59:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:59:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:59:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790371,"replay_state":"idle"} 2026-02-22T19:59:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:34 2026-02-22T19:59:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:59:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 80 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.e91e58e8-3431-42c6-bb9d-3b2dc1cc1ab1 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T19:59:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:59:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:59:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:59:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:59:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:59:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:38' 2026-02-22T19:59:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 19:59:38' 2026-02-22T19:59:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:38' 2026-02-22T19:59:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:40.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 19:59:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790371,"replay_state":"idle"}2026-02-22 19:59:34' 2026-02-22T19:59:40.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:40.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:40.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:38' 2026-02-22T19:59:40.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:40.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:59:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:59:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:40.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:40.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:41.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T19:59:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:59:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790371,"replay_state":"idle"} 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:34 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:59:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:38' 2026-02-22T19:59:41.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:59:41.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:59:41.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790371,"replay_state":"idle"} 2026-02-22T19:59:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:59:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:41.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:59:41.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:41.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:34' 2026-02-22T19:59:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 19:59:34' 2026-02-22T19:59:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:34' 2026-02-22T19:59:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790371,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 19:59:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 19:59:38' 2026-02-22T19:59:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:34' 2026-02-22T19:59:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:41.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:59:41.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:59:41.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:59:41.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:59:41.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:41.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:41.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T19:59:41.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T19:59:41.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:41.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:41.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:41.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T19:59:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:59:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:59:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:59:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:59:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:59:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:59:43.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:59:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:59:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:59:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:43.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:43.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:43.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:59:43.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:59:43.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:59:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:59:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:59:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:59:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T19:59:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:59:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T19:59:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T19:59:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T19:59:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T19:59:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T19:59:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T19:59:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T19:59:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T19:59:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T19:59:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T19:59:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T19:59:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:45.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:59:45.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:59:45.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:59:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:59:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:46.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T19:59:46.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:46.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:46.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:46.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:46.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:59:46.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:44 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:46.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:43' 2026-02-22T19:59:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:59:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:59:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:59:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:46.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:46.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:46.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:44' 2026-02-22T19:59:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:59:44' 2026-02-22T19:59:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:44' 2026-02-22T19:59:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:46.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:59:44cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 19:59:43' 2026-02-22T19:59:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:44' 2026-02-22T19:59:46.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:59:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:59:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T19:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T19:59:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:47.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:47.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:47.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:47.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:47.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:47.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:43 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T19:59:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T19:59:47.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:44' 2026-02-22T19:59:47.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T19:59:47.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T19:59:47.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:47.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T19:59:47.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:47.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:47.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:47.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:47.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:43' 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:59:43' 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:43' 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 19:59:43cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:59:44' 2026-02-22T19:59:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:47.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:47.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:43' 2026-02-22T19:59:47.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:47.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T19:59:47.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T19:59:47.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T19:59:47.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T19:59:47.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:47.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:47.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T19:59:47.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:47.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:47.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:47.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T19:59:47.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T19:59:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T19:59:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:59:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:49.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T19:59:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T19:59:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:59:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:59:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:59:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:59:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:59:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:59:49.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:59:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:59:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:59:49.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:59:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:59:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:59:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:59:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T19:59:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T19:59:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:59:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T19:59:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T19:59:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T19:59:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T19:59:53.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 83 2026-02-22T19:59:53.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T19:59:53.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T19:59:53.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:53.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:53.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:53.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T19:59:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T19:59:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:59:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T19:59:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T19:59:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T19:59:53.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:53.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:53.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:53.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:53.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:53.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T19:59:53.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T19:59:53.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:59:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T19:59:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=83 2026-02-22T19:59:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T19:59:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T19:59:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T19:59:54.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:54.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:54.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:54.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:54.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T19:59:54.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T19:59:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T19:59:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T19:59:54.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=83 2026-02-22T19:59:54.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 83 = 83 2026-02-22T19:59:54.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:54.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T19:59:54.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:54.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:54.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:54.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:54.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:54.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:54.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:54.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:55.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T19:59:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T19:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:54 2026-02-22T19:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T19:59:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:59:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790387,"replay_state":"idle"} 2026-02-22T19:59:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:48 2026-02-22T19:59:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T19:59:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 83 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.aeeebba0-0617-4566-b9db-80652fd81874 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T19:59:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T19:59:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T19:59:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T19:59:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:59:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:55.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:59:55.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:55.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:55.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:55.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:54' 2026-02-22T19:59:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 19:59:54' 2026-02-22T19:59:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T19:59:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:54' 2026-02-22T19:59:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:55.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 19:59:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790387,"replay_state":"idle"}2026-02-22 19:59:48' 2026-02-22T19:59:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:54' 2026-02-22T19:59:55.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T19:59:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T19:59:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T19:59:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T19:59:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T19:59:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:55.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:55.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:55.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:55.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T19:59:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:56.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:56.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:56.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:56.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:56.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:56.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:56.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:56.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T19:59:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T19:59:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T19:59:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T19:59:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790387,"replay_state":"idle"} 2026-02-22T19:59:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T19:59:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:48 2026-02-22T19:59:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T19:59:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T19:59:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T19:59:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T19:59:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:54' 2026-02-22T19:59:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T19:59:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T19:59:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T19:59:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790387,"replay_state":"idle"} 2026-02-22T19:59:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T19:59:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:59:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T19:59:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:59:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T19:59:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T19:59:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T19:59:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T19:59:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:48' 2026-02-22T19:59:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 19:59:48' 2026-02-22T19:59:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T19:59:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:48' 2026-02-22T19:59:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T19:59:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T19:59:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T19:59:56.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790387,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 19:59:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 19:59:54' 2026-02-22T19:59:56.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T19:59:56.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T19:59:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:48' 2026-02-22T19:59:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T19:59:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T19:59:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T19:59:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T19:59:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T19:59:56.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:56.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T19:59:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T19:59:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T19:59:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T19:59:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T19:59:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T19:59:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T19:59:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T19:59:57.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:59:57.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:57.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:57.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:57.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:59:57.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:57.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:59:57.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:57.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:57.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:59:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:59:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:59:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T19:59:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T19:59:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:57.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T19:59:57.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T19:59:57.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T19:59:57.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:57.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T19:59:59.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T19:59:59.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T19:59:59.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:59.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:59.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T19:59:59.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T19:59:59.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T19:59:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T19:59:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T19:59:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T19:59:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T19:59:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T19:59:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T19:59:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T19:59:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T19:59:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T19:59:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T19:59:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T19:59:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T19:59:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T19:59:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T19:59:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T19:59:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T19:59:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T19:59:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T19:59:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T19:59:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T19:59:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T19:59:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T19:59:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T19:59:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T19:59:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T19:59:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T19:59:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T19:59:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T19:59:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T19:59:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T19:59:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T19:59:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T19:59:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T19:59:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T19:59:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:00:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:00:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:00:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:59 2026-02-22T20:00:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:00:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:00:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:58' 2026-02-22T20:00:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:00:00.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:00:00.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:00:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:00.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:00.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:00.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:59' 2026-02-22T20:00:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 19:59:59' 2026-02-22T20:00:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:59' 2026-02-22T20:00:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 19:59:59cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 19:59:58' 2026-02-22T20:00:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:59' 2026-02-22T20:00:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:00:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:00:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:00:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:00:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:00:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:00:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:58 2026-02-22T20:00:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:00:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:00:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 19:59:59' 2026-02-22T20:00:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:00:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:00:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:00:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:58' 2026-02-22T20:00:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 19:59:58' 2026-02-22T20:00:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 19:59:58' 2026-02-22T20:00:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted44434452mirror.0trial0982026-02-22 19:59:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 19:59:59' 2026-02-22T20:00:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 19:59:58' 2026-02-22T20:00:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:00:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:00:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:02.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:02.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:02.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:00:02.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:02.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:02.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:00:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:00:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:00:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:00:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:00:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:00:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:00:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:00:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:00:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:00:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:00:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:00:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:00:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:00:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:00:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:00:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:00:04.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:00:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:00:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:00:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:00:06.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:00:06.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:06.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:06.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:06.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:00:06.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:00:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:00:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:00:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:00:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:06.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:00:06.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:06.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:06.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:06.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:00:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:00:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:00:06.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:00:06.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:00:06.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:00:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 86 2026-02-22T20:00:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:00:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:00:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:00:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:00:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:00:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:00:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:00:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:00:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:00:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:00:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:00:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:00:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:00:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:00:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:00:10.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=86 2026-02-22T20:00:10.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:00:10.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:00:10.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:00:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:00:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:00:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:00:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:00:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:00:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=86 2026-02-22T20:00:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 86 = 86 2026-02-22T20:00:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:00:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:00:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:00:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:00:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:00:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:00:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:00:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:08 2026-02-22T20:00:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:00:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:00:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790402,"replay_state":"idle"} 2026-02-22T20:00:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:04 2026-02-22T20:00:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:00:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 86 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.7cf04d91-d1b5-484e-a995-80a64f84aeef (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:00:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:00:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:00:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:00:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:00:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:11.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:00:11.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:08' 2026-02-22T20:00:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:00:08' 2026-02-22T20:00:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:00:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:08' 2026-02-22T20:00:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:00:08cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790402,"replay_state":"idle"}2026-02-22 20:00:04' 2026-02-22T20:00:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:08' 2026-02-22T20:00:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:00:11.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:00:11.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:00:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:00:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:00:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:00:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:11.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:00:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:00:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:00:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:00:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790402,"replay_state":"idle"} 2026-02-22T20:00:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:04 2026-02-22T20:00:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:00:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:00:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:00:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:08' 2026-02-22T20:00:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:00:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:00:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790402,"replay_state":"idle"} 2026-02-22T20:00:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:00:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:00:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:12.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:04' 2026-02-22T20:00:12.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:00:04' 2026-02-22T20:00:12.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:00:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:04' 2026-02-22T20:00:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:12.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790402,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:00:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:00:08' 2026-02-22T20:00:12.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:12.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:12.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:04' 2026-02-22T20:00:12.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:12.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:00:12.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:00:12.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:00:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:00:12.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:12.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:12.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:00:12.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:00:12.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:12.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:12.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:12.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:00:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:00:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:00:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:00:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:00:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:00:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:14.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:00:14.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:14.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:14.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:14.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:00:14.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:14.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:00:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:00:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:00:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:00:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:14.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:14.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:00:14.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:14.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:14.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:00:14.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:14.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:00:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:00:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:00:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:00:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:14.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:00:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:00:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:00:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:00:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:00:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:00:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:16.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:00:16.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:16.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:00:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:00:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:00:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:00:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:00:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:00:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:16.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:00:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:16.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:16.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:16.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:00:16.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:00:16.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:00:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:00:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:00:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:00:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:00:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:14 2026-02-22T20:00:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:00:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:00:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:00:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:13' 2026-02-22T20:00:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:00:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:00:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:00:17.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:17.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:17.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:14' 2026-02-22T20:00:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:00:14' 2026-02-22T20:00:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:14' 2026-02-22T20:00:17.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:17.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:17.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:17.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 20:00:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:00:13' 2026-02-22T20:00:17.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:17.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:17.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:14' 2026-02-22T20:00:17.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:17.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:00:17.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:00:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:00:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:00:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:00:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:00:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:18 2026-02-22T20:00:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:00:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:00:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:14' 2026-02-22T20:00:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:00:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:00:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:00:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:18' 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:00:18' 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:18' 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:18.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:00:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 20:00:14' 2026-02-22T20:00:18.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:18.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:18.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:18' 2026-02-22T20:00:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:00:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:00:18.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:00:18.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:00:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:00:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:00:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:00:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:00:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:00:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:00:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:00:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:00:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:00:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:00:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:00:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:00:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:00:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:00:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:00:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:20.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:00:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:00:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:00:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:00:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:00:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:00:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:00:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:00:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:00:22.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:00:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:00:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:00:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:00:22.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:22.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:22.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:22.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:00:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:00:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:00:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:00:26.936 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:00:26.936 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:26.936 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:00:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:00:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:00:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:00:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:00:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:00:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:00:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:00:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:00:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:00:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:00:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:00:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:00:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:00:27.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:00:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 89 2026-02-22T20:00:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:00:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:00:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:00:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:00:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:00:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:00:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:00:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:00:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:00:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:00:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:00:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:00:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:00:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:00:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:00:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=89 2026-02-22T20:00:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:00:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:00:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:00:31.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:00:31.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:31.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:31.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:31.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:00:31.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:00:31.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:00:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:00:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=89 2026-02-22T20:00:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 89 = 89 2026-02-22T20:00:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:00:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:00:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:00:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:00:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:32.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:32.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:32.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:00:32.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:32.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:32.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:00:32.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:00:32.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:32.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:29 2026-02-22T20:00:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:00:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:00:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790418,"replay_state":"idle"} 2026-02-22T20:00:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:23 2026-02-22T20:00:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:00:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 89 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.ef71aa45-ce93-44d0-9f3f-ae06ba253941 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:00:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:00:32.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:00:32.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:32.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:00:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:00:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:00:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:29' 2026-02-22T20:00:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:00:29' 2026-02-22T20:00:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:00:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:29' 2026-02-22T20:00:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:00:29cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790418,"replay_state":"idle"}2026-02-22 20:00:23' 2026-02-22T20:00:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:32.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:29' 2026-02-22T20:00:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:00:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:00:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:00:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:00:32.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:32.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:32.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:00:32.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:32.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:32.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:32.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:00:32.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:32.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:33.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:00:33.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:33.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:33.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:33.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:00:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:00:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:00:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790418,"replay_state":"idle"} 2026-02-22T20:00:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:23 2026-02-22T20:00:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:33.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:00:33.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:00:33.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:00:33.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:29' 2026-02-22T20:00:33.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:00:33.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:00:33.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:33.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790418,"replay_state":"idle"} 2026-02-22T20:00:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:00:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:33.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:00:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:33.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:33.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:33.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:33.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:23' 2026-02-22T20:00:33.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:00:23' 2026-02-22T20:00:33.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:33.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:33.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:33.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:00:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:23' 2026-02-22T20:00:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790418,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:00:23cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:00:29' 2026-02-22T20:00:33.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:33.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:23' 2026-02-22T20:00:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:33.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:00:33.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:00:33.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:00:33.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:00:33.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:33.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:33.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:00:33.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:33.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:33.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:00:33.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:00:33.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:00:33.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:33.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:33.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:00:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:00:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:00:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:34.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:00:34.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:34.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:34.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:34.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:00:34.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:34.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:34.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:00:34.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:34.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:00:34.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:34.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:00:34.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:00:34.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:34.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:34.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:00:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:00:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:00:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:00:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:00:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:00:34.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:00:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:00:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:00:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:00:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:00:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:00:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:00:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:00:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:00:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:00:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:00:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:00:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:36.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:00:36.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:36.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:36.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:00:36.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:00:36.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:36.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:36.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:00:36.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:00:36.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:00:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:36.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:36.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:00:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:36.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:36.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:36.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:00:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:00:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:00:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:00:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:00:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:37.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:37.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:37.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:37.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:37.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:00:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:00:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:34 2026-02-22T20:00:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:00:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:00:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:35' 2026-02-22T20:00:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:00:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:00:37.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:37.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:00:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:34' 2026-02-22T20:00:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:00:34' 2026-02-22T20:00:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:34' 2026-02-22T20:00:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:00:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 20:00:35' 2026-02-22T20:00:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:34' 2026-02-22T20:00:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:00:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:00:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:00:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:00:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:00:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:00:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:38 2026-02-22T20:00:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:00:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:00:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:34' 2026-02-22T20:00:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:00:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:00:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:00:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:39.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:38' 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:00:38' 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:38' 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:00:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:00:34' 2026-02-22T20:00:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:38' 2026-02-22T20:00:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:00:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:00:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:00:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:00:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:00:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:00:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:00:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:00:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:00:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:00:41.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:00:41.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:41.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:41.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:41.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:00:41.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:00:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:00:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:00:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:00:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:00:41.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:41.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:41.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:41.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:41.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:41.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:00:41.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:00:41.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:00:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:41.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:41.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:41.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:00:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 92 2026-02-22T20:00:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:00:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:00:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:00:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:00:45.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:00:45.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:00:45.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:00:45.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:00:45.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:00:45.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:00:45.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:00:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:00:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:00:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:00:45.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:00:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=92 2026-02-22T20:00:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:00:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:00:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:00:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:00:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:00:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:00:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:00:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:00:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=92 2026-02-22T20:00:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 92 = 92 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:46.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:00:46.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:46.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:00:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:00:46.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:46.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:46.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:00:46.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:43 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790439,"replay_state":"idle"} 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:39 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 92 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.bf38fbf4-e759-4ca3-ae06-5366a6a8e5c5 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:00:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:00:46.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:00:46.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:00:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:00:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:00:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:43' 2026-02-22T20:00:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:00:43' 2026-02-22T20:00:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:00:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:43' 2026-02-22T20:00:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:00:43cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790439,"replay_state":"idle"}2026-02-22 20:00:39' 2026-02-22T20:00:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:43' 2026-02-22T20:00:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:00:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:00:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:00:46.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:00:46.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:46.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:46.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:00:46.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:46.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:00:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:00:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:00:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:47.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:47.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:00:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:00:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790439,"replay_state":"idle"} 2026-02-22T20:00:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:39 2026-02-22T20:00:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:00:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:00:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:00:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:43' 2026-02-22T20:00:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:00:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:00:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790439,"replay_state":"idle"} 2026-02-22T20:00:47.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:47.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:00:47.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:00:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:39' 2026-02-22T20:00:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:00:39' 2026-02-22T20:00:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:00:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:39' 2026-02-22T20:00:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:47.608 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:47.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790439,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:00:39cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:00:43' 2026-02-22T20:00:47.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:47.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:47.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:39' 2026-02-22T20:00:47.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:47.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:00:47.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:00:47.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:00:47.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:00:47.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:47.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:47.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:00:47.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:00:47.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:47.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:47.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:47.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:00:48.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:00:48.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:00:48.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:48.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:00:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:00:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:00:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:00:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:00:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:00:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:00:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:00:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:00:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:00:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:00:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:00:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:00:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:00:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:00:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:00:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:00:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:00:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:00:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:00:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:00:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:00:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:00:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:00:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:00:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:00:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:00:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:00:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:00:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:00:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:00:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:00:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:00:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:00:51.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:51.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:00:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:00:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:51.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:00:51.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:00:51.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:00:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:00:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:00:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:49 2026-02-22T20:00:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:00:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:00:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:00:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:48' 2026-02-22T20:00:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:00:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:00:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:52.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:52.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:49' 2026-02-22T20:00:52.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:00:49' 2026-02-22T20:00:52.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:52.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:52.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:49' 2026-02-22T20:00:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 20:00:49cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:00:48' 2026-02-22T20:00:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:49' 2026-02-22T20:00:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:52.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:00:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:00:52.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:00:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:00:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:00:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:00:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:00:53.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:00:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:00:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:00:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:00:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:53 2026-02-22T20:00:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:00:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:00:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:00:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:00:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:49' 2026-02-22T20:00:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:00:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:00:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:00:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:00:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:00:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:00:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:00:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:00:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:00:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:00:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:53' 2026-02-22T20:00:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:00:53' 2026-02-22T20:00:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:00:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:53' 2026-02-22T20:00:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:00:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:00:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:00:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:00:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 20:00:49' 2026-02-22T20:00:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:00:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:00:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:53' 2026-02-22T20:00:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:00:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:00:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:00:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:00:53.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:00:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:00:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:00:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:00:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:00:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:00:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:00:54.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:54.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:00:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:00:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:00:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:55.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:00:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:00:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:00:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:00:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:00:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:00:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:00:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:00:55.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:00:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:00:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:00:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:00:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:00:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:00:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:00:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:00:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:00:57.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:00:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:00:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:00:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:00:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:00:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:00:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:00:57.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:00:57.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:00:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:00:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:00:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:00:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:00:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:00:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:00:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:00:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:00:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:00:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:00:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:00:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:00:57.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:00:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:00:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:00:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:00:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:00:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:00:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:01:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:01:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:01:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:01:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:01:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:01:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:01:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:01:01.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:01:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:01:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:01:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:01:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:01:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:01:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:01:02.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:01:02.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:02.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:01:02.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:01:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 95 2026-02-22T20:01:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:01:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:01:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:01:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:01:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:01:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:01:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:01:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:01:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:01:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:01:05.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:01:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=95 2026-02-22T20:01:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:01:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:01:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:01:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:01:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:01:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:01:06.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=95 2026-02-22T20:01:06.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 95 = 95 2026-02-22T20:01:06.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:01:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:01:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:01:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:01:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:01:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:04 2026-02-22T20:01:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:01:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:01:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790453,"replay_state":"idle"} 2026-02-22T20:01:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:58 2026-02-22T20:01:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:01:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 95 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.ca893f85-cb56-4abb-9680-ed6d1ac6233a (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:01:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:01:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:01:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:01:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:01:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:01:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:04' 2026-02-22T20:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:01:04' 2026-02-22T20:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:04' 2026-02-22T20:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:01:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790453,"replay_state":"idle"}2026-02-22 20:00:58' 2026-02-22T20:01:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:07.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:04' 2026-02-22T20:01:07.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:07.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:01:07.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:01:07.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:01:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:01:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:01:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790453,"replay_state":"idle"} 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:00:58 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:04' 2026-02-22T20:01:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:01:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:01:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790453,"replay_state":"idle"} 2026-02-22T20:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:58' 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:00:58' 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:00:58' 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790453,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:00:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:01:04' 2026-02-22T20:01:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:08.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:00:58' 2026-02-22T20:01:08.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:01:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:01:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:01:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:01:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:01:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:01:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:01:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:01:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:01:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:01:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:01:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:09.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:01:09.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:09.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:09.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:01:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:01:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:01:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:01:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:01:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:01:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:01:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:01:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:09.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:01:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:01:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:01:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:01:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:01:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:01:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:01:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:01:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:01:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:01:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:01:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:01:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:01:11.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:01:11.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:11.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:01:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:01:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:01:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:11.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:11.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:11.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:01:11.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:01:11.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:11.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:11.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:01:11.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:11.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:11.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:11.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:11.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:01:11.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:01:11.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:01:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:01:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:01:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:09 2026-02-22T20:01:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:01:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:01:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:10' 2026-02-22T20:01:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:01:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:01:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:01:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:12.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:09' 2026-02-22T20:01:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:01:09' 2026-02-22T20:01:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:09' 2026-02-22T20:01:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:01:09cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 20:01:10' 2026-02-22T20:01:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:13.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:09' 2026-02-22T20:01:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:13.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:01:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:01:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:01:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:01:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:01:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:01:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:13 2026-02-22T20:01:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:01:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:01:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:09' 2026-02-22T20:01:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:01:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:01:14.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:14.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:01:14.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:14.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:13' 2026-02-22T20:01:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:01:13' 2026-02-22T20:01:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:13' 2026-02-22T20:01:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:01:13cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:01:09' 2026-02-22T20:01:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:14.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:13' 2026-02-22T20:01:14.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:01:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:14.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:01:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:01:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:01:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:01:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:01:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:01:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:01:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:01:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:01:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:01:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:01:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:16.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:01:16.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:16.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:16.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:01:16.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:01:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:01:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:01:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:01:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:01:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:01:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:01:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:01:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:01:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:01:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:01:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:01:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:01:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:01:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:01:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:01:16.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:01:16.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:01:16.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:16.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:16.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:16.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 98 2026-02-22T20:01:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:01:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:01:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:01:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:01:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:01:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:01:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:01:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:01:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:01:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:01:20.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:01:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=98 2026-02-22T20:01:20.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:01:20.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:01:20.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:01:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:01:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:01:20.695 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:01:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=98 2026-02-22T20:01:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 98 = 98 2026-02-22T20:01:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:01:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:01:21.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:21.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:21.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:01:21.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:18 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790474,"replay_state":"idle"} 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:14 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 98 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.94502d25-da86-4ee1-90b8-f40b1352a825 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:01:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:01:21.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:01:21.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:21.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:01:21.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:21.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:01:21.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:21.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:01:21.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:21.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:21.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:21.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:18' 2026-02-22T20:01:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:01:18' 2026-02-22T20:01:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:18' 2026-02-22T20:01:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:21.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:01:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790474,"replay_state":"idle"}2026-02-22 20:01:14' 2026-02-22T20:01:21.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:21.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:18' 2026-02-22T20:01:21.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:21.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:01:21.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:01:21.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:01:21.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:01:21.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:21.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:21.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:01:21.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:21.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:21.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:21.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:21.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:21.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:21.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:21.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:01:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:01:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:01:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790474,"replay_state":"idle"} 2026-02-22T20:01:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:14 2026-02-22T20:01:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:01:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:01:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:01:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:18' 2026-02-22T20:01:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:01:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:01:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790474,"replay_state":"idle"} 2026-02-22T20:01:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:01:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:01:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:14' 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:01:14' 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:14' 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790474,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:01:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:01:18' 2026-02-22T20:01:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:14' 2026-02-22T20:01:22.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:01:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:01:22.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:01:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:01:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:01:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:01:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:22.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:01:23.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:01:23.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:01:23.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:23.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:23.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:23.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:01:23.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:23.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:23.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:23.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:01:23.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:01:23.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:23.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:01:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:01:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:01:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:01:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:01:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:01:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:24.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:24.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:01:24.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:01:24.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:24.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:01:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:01:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:01:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:01:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:01:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:01:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:01:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:01:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:01:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:01:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:01:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:01:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:01:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:01:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:01:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:01:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:01:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:01:27.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:24 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:23' 2026-02-22T20:01:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:01:27.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:01:27.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:27.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:01:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:24' 2026-02-22T20:01:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:01:24' 2026-02-22T20:01:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:24' 2026-02-22T20:01:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:27.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 20:01:24cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:01:23' 2026-02-22T20:01:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:24' 2026-02-22T20:01:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:01:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:01:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:01:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:01:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:01:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:28 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:24' 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:01:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:01:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:01:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:28.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:28.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:28' 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:01:28' 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:28' 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:01:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 20:01:24' 2026-02-22T20:01:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:28.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:28' 2026-02-22T20:01:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:28.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:01:28.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:28.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:01:28.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:01:28.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:28.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:28.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:28.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:01:28.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:01:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:01:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:01:29.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:29.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:01:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:01:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:01:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:01:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:30.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:30.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:01:30.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:01:30.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:01:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:01:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:01:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:01:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:32.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:32.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:01:32.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:01:32.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:01:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:01:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:01:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:01:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:01:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:01:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:36.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:36.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:01:36.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:36.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:01:36.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:36.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:01:36.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:01:36.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:36.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:36.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:36.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:01:36.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:01:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:01:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:01:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:01:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:01:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:01:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:01:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:01:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:01:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:01:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:01:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:01:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:01:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:01:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:37.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:01:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:01:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:01:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:01:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:01:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:01:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 101 2026-02-22T20:01:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:01:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:01:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:01:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:01:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:01:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:01:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:01:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:01:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:01:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=101 2026-02-22T20:01:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:01:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:01:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:01:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:01:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:01:41.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:01:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=101 2026-02-22T20:01:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 101 = 101 2026-02-22T20:01:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:39 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790488,"replay_state":"idle"} 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:33 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 101 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.4e8e04a7-ec29-4e8d-9409-f1d452c58928 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:01:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:01:42.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:01:42.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:42.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:01:42.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:42.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:01:42.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:42.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:01:42.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:39' 2026-02-22T20:01:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:01:39' 2026-02-22T20:01:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:39' 2026-02-22T20:01:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:01:39cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790488,"replay_state":"idle"}2026-02-22 20:01:33' 2026-02-22T20:01:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:39' 2026-02-22T20:01:42.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:01:42.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:01:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:01:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:01:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790488,"replay_state":"idle"} 2026-02-22T20:01:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:33 2026-02-22T20:01:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:01:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:01:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:01:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:39' 2026-02-22T20:01:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:01:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:01:43.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790488,"replay_state":"idle"} 2026-02-22T20:01:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:01:43.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:43.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:01:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:33' 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:01:33' 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:33' 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790488,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:01:33cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:01:39' 2026-02-22T20:01:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:43.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:33' 2026-02-22T20:01:43.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:01:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:01:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:01:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:01:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:01:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:01:43.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:01:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:01:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:01:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:01:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:01:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:01:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:01:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:01:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:01:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:01:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:01:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:44.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:01:44.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:44.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:01:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:01:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:01:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:01:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:01:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:01:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:01:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:01:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:01:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:01:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:01:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:01:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:01:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:01:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:01:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:01:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:01:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:01:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:01:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:01:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:01:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:01:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:01:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:01:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:01:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:01:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:44 2026-02-22T20:01:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:01:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:01:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:45' 2026-02-22T20:01:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:01:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:01:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:01:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:47.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:47.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:44' 2026-02-22T20:01:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:01:44' 2026-02-22T20:01:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:44' 2026-02-22T20:01:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:48.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:01:44cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 20:01:45' 2026-02-22T20:01:48.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:48.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:48.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:44' 2026-02-22T20:01:48.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:48.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:01:48.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:48.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:01:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:01:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:01:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:48 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:44' 2026-02-22T20:01:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:01:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:01:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:01:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:48' 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:01:48' 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:48' 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:01:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:01:44' 2026-02-22T20:01:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:48' 2026-02-22T20:01:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:01:49.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:01:49.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:01:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:01:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:49.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:01:49.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:49.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:49.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:49.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:01:49.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:01:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:01:50.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:01:50.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:01:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:01:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:01:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:01:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:01:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:01:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:01:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:01:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:01:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:01:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:01:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:01:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:01:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:01:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:01:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:01:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:01:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:01:51.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:51.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:51.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:51.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:01:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:01:51.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:51.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:01:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:01:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:01:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:01:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:01:55.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 104 2026-02-22T20:01:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:01:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:01:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:01:55.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:01:55.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:01:55.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:01:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:01:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:01:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:01:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:01:55.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:01:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:01:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=104 2026-02-22T20:01:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:01:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:01:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:01:55.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:55.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:55.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:55.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:55.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:01:55.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:01:55.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:01:55.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:01:55.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=104 2026-02-22T20:01:55.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 104 = 104 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:01:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:53 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790512,"remote_snapshot_timestamp":1771790512,"replay_state":"idle"} 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:54 2026-02-22T20:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:01:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 104 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.fa47b97e-77d6-4870-9f1e-d57eabea04bf (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:01:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:01:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:01:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:56.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:01:56.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:56.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:01:56.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:01:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:53' 2026-02-22T20:01:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:01:53' 2026-02-22T20:01:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:01:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:53' 2026-02-22T20:01:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:01:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790512,"remote_snapshot_timestamp":1771790512,"replay_state":"idle"}2026-02-22 20:01:54' 2026-02-22T20:01:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:53' 2026-02-22T20:01:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:01:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:01:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:01:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:01:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:01:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:56.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:01:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:01:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:01:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:01:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:01:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:01:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:01:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790512,"remote_snapshot_timestamp":1771790512,"replay_state":"idle"} 2026-02-22T20:01:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:01:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:54 2026-02-22T20:01:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:01:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:01:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:01:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:01:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:53' 2026-02-22T20:01:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:01:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:01:57.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:01:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790512,"remote_snapshot_timestamp":1771790512,"replay_state":"idle"} 2026-02-22T20:01:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:01:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:01:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:01:57.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:01:57.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:01:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:01:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:01:57.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:01:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:54' 2026-02-22T20:01:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:01:54' 2026-02-22T20:01:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:01:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:54' 2026-02-22T20:01:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:01:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:01:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:01:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790512,"remote_snapshot_timestamp":1771790512,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:01:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:01:53' 2026-02-22T20:01:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:01:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:01:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:54' 2026-02-22T20:01:57.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:01:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:01:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:01:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:01:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:01:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:01:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:01:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:01:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:01:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:01:58.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:01:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:01:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:01:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:01:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:01:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:01:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:01:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:01:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:01:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:01:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:01:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:01:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:01:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:01:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:01:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:01:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:01:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:01:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:01:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:01:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:01:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:01:59.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:01:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:01:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:01:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:01:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:01:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:01:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:01:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:01:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:02:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:02:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:02:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:02:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:02:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:02:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:02:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:02:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:02:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:02:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:02:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:02:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:02:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:02:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:02:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:02:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:02:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:02.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:02.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:02:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:02:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:59 2026-02-22T20:02:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:02:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:02:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:02:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:58' 2026-02-22T20:02:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:02:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:02:02.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:02:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:02.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:02.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:02.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:59' 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:01:59' 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:01:59' 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 20:01:59cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:01:58' 2026-02-22T20:02:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:02.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:01:59' 2026-02-22T20:02:02.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:02:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:02.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:02:02.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:02:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:03.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:02:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:03 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:01:59' 2026-02-22T20:02:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:02:03.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:02:03.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:03.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:02:03.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:03.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:03.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:03.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:03.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:03.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:03' 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:02:03' 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:03' 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:02:03cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 20:01:59' 2026-02-22T20:02:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:03' 2026-02-22T20:02:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:03.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:02:03.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:03.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:02:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:02:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:02:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:02:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:02:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:02:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:02:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:02:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:05.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:05.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:05.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:02:05.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:02:05.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:05.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:02:05.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:05.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:02:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:02:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:07.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:07.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:07.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:07.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:02:07.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:02:07.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:02:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:02:11.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:02:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:02:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:11.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:11.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:11.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:11.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:11.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:11.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:12.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:12.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:02:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:02:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:02:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:12.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:12.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:12.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:02:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 107 2026-02-22T20:02:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:02:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:02:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:02:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:02:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:02:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:02:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:02:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:02:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:02:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:02:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:02:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:02:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:02:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:02:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:02:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=107 2026-02-22T20:02:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:02:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:02:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:02:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:02:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:02:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:02:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:02:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:02:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=107 2026-02-22T20:02:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 107 = 107 2026-02-22T20:02:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:02:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:02:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:02:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:02:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:02:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:02:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:02:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:14 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790523,"replay_state":"idle"} 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:08 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 107 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.3c9f47c0-1ae7-410d-8e01-9d0565898c96 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:02:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:02:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:02:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:02:17.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:17.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:02:17.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:17.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:17.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:17.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:14' 2026-02-22T20:02:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:02:14' 2026-02-22T20:02:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:02:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:14' 2026-02-22T20:02:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:02:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790523,"replay_state":"idle"}2026-02-22 20:02:08' 2026-02-22T20:02:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:14' 2026-02-22T20:02:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:02:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:02:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:02:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:02:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:02:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:02:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:02:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:02:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790532,"remote_snapshot_timestamp":1771790532,"replay_state":"idle"} 2026-02-22T20:02:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:18 2026-02-22T20:02:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:02:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:02:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:02:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:14' 2026-02-22T20:02:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:02:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:02:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790532,"remote_snapshot_timestamp":1771790532,"replay_state":"idle"} 2026-02-22T20:02:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:02:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:02:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:18.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:18' 2026-02-22T20:02:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:02:18' 2026-02-22T20:02:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:02:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:18' 2026-02-22T20:02:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:18.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790532,"remote_snapshot_timestamp":1771790532,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:02:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:02:14' 2026-02-22T20:02:18.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:18.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:18.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:18' 2026-02-22T20:02:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:18.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:02:18.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:02:18.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:02:18.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:02:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:02:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:02:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:02:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:02:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:19.818 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:19.818 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:19.818 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:02:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:02:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:02:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:02:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:02:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:02:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:02:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:02:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:21.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:21.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:02:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:02:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:02:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:02:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:21.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:21.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:21.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:02:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:19 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:20' 2026-02-22T20:02:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:02:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:02:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:02:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:19' 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:02:19' 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:19' 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:23.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:02:19cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 20:02:20' 2026-02-22T20:02:23.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:23.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:19' 2026-02-22T20:02:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:02:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:02:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:02:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:02:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:02:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:02:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:23 2026-02-22T20:02:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:02:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:02:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:19' 2026-02-22T20:02:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:02:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:02:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:02:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:24.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:23' 2026-02-22T20:02:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:02:23' 2026-02-22T20:02:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:23' 2026-02-22T20:02:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:02:23cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:02:19' 2026-02-22T20:02:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:23' 2026-02-22T20:02:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:02:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:02:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:02:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:02:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:02:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:02:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:02:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:02:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:02:26.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:26.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:02:26.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:02:26.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:26.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:26.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:02:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:02:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:02:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:26.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:26.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:02:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:02:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:02:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 110 2026-02-22T20:02:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:02:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:02:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:02:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:02:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:02:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:02:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:02:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:02:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:02:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:02:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:02:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:02:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:02:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:02:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:02:30.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=110 2026-02-22T20:02:30.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:02:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:02:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:02:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:02:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:02:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:02:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:02:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:02:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=110 2026-02-22T20:02:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 110 = 110 2026-02-22T20:02:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:02:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:02:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:02:31.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:31.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:31.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:02:31.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:02:31.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:28 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790544,"replay_state":"idle"} 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:24 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 110 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.597ca65b-3b57-43bc-9861-1a704a516b69 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:02:31.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:02:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:02:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:02:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:28' 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:02:28' 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:28' 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:31.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:02:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790544,"replay_state":"idle"}2026-02-22 20:02:24' 2026-02-22T20:02:31.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:31.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:31.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:28' 2026-02-22T20:02:31.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:31.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:02:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:02:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:02:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:02:31.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:02:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:02:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790544,"replay_state":"idle"} 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:24 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:28' 2026-02-22T20:02:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:02:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:02:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790544,"replay_state":"idle"} 2026-02-22T20:02:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:02:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:02:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:24' 2026-02-22T20:02:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:02:24' 2026-02-22T20:02:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:02:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:24' 2026-02-22T20:02:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790544,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:02:24cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:02:28' 2026-02-22T20:02:32.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:32.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:24' 2026-02-22T20:02:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:02:32.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:02:32.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:02:32.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:02:32.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:32.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:32.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:02:32.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:02:32.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:32.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:32.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:32.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:02:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:02:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:02:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:02:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:34.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:34.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:02:34.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:02:34.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:34.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:02:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:02:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:02:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:02:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:02:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:02:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:02:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:02:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:02:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:02:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:02:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:36.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:36.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:02:36.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:02:36.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:36.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:36.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:02:36.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:02:36.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:36.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:02:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:02:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:02:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:02:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:36.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:02:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:02:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:02:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:02:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:37.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:02:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:02:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:02:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:34 2026-02-22T20:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:33' 2026-02-22T20:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:02:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:02:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:34' 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:02:34' 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:34' 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:37.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 20:02:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:02:33' 2026-02-22T20:02:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:34' 2026-02-22T20:02:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:02:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:02:37.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:02:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:38.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:02:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:38.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:02:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:02:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:38 2026-02-22T20:02:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:02:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:02:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:34' 2026-02-22T20:02:38.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:02:38.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:02:38.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:38.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:02:38.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:38.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:38.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:38.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:38.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:38' 2026-02-22T20:02:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:02:38' 2026-02-22T20:02:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:38' 2026-02-22T20:02:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:02:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 20:02:34' 2026-02-22T20:02:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:38.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:38' 2026-02-22T20:02:38.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:38.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:02:38.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:38.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:02:38.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:38.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:38.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:02:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:02:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:02:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:02:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:02:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:02:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:02:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:02:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:02:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:02:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:40.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:40.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:40.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:40.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:02:40.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:02:40.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:02:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:02:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:02:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:42.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:42.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:02:42.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:42.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:42.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:42.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:42.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:42.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:42.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:42.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:02:42.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:02:42.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:42.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:02:42.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:42.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:02:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:46.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:46.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:46.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:46.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:47.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:02:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:02:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:02:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:02:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:02:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 113 2026-02-22T20:02:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:02:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:02:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:02:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:02:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:02:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:02:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:02:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:02:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:02:50.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:02:50.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:02:50.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:50.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:50.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:50.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:02:50.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:02:50.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:02:50.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:02:50.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=113 2026-02-22T20:02:50.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:02:50.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:02:50.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:02:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:02:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:02:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:02:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:02:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:02:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=113 2026-02-22T20:02:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 113 = 113 2026-02-22T20:02:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:02:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:02:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:02:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:02:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:02:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:02:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:02:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:49 2026-02-22T20:02:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:02:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:02:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790558,"remote_snapshot_timestamp":1771790558,"replay_state":"idle"} 2026-02-22T20:02:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:48 2026-02-22T20:02:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:02:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 113 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.68df2f12-70c6-4db1-bbe3-88badb37e932 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:02:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:02:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:02:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:02:52.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:52.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:02:52.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:52.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:02:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:52.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:52.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:52.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:49' 2026-02-22T20:02:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:02:49' 2026-02-22T20:02:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:02:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:49' 2026-02-22T20:02:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:02:49cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790558,"remote_snapshot_timestamp":1771790558,"replay_state":"idle"}2026-02-22 20:02:48' 2026-02-22T20:02:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:49' 2026-02-22T20:02:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:02:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:02:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:02:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:02:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790558,"remote_snapshot_timestamp":1771790558,"replay_state":"idle"} 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:48 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:49' 2026-02-22T20:02:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:02:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:02:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790558,"remote_snapshot_timestamp":1771790558,"replay_state":"idle"} 2026-02-22T20:02:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:02:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:02:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:48' 2026-02-22T20:02:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:02:48' 2026-02-22T20:02:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:02:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:48' 2026-02-22T20:02:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:53.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:53.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790558,"remote_snapshot_timestamp":1771790558,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:02:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:02:49' 2026-02-22T20:02:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:48' 2026-02-22T20:02:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:02:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:02:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:02:53.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:02:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:02:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:02:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:53.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:53.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:02:53.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:02:53.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:53.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:54.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:02:54.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:54.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:54.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:54.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:02:54.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:54.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:54.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:54.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:54.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:54.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:54.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:54.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:54.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:54.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:54.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:54.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:54.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:54.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:54.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:02:54.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:02:54.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:54.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:02:54.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:02:54.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:02:54.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:54.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:02:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:02:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:02:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:02:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:02:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:02:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:02:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:02:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:02:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:02:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:02:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:02:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:02:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:02:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:02:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:02:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:02:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:02:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:02:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:02:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:02:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:02:56.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:02:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:02:56.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:02:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:02:56.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:02:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:02:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:02:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:02:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:02:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:02:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:02:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:54 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:58' 2026-02-22T20:02:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:02:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:02:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:02:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:54' 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:02:54' 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:54' 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:02:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:02:58' 2026-02-22T20:02:58.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:58.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:54' 2026-02-22T20:02:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:02:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:02:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:02:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:02:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:02:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:02:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:02:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:02:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:02:59.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:02:59.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:02:59.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:59.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:02:59.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:02:59.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:58 2026-02-22T20:02:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:02:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:02:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:02:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:02:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:02:54' 2026-02-22T20:02:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:02:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:02:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:02:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:02:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:02:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:02:59.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:59.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:02:59.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:02:59.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:02:59.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:02:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:58' 2026-02-22T20:02:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:02:58' 2026-02-22T20:02:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:02:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:02:58' 2026-02-22T20:02:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:02:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:02:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:02:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:02:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:02:59' 2026-02-22T20:02:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:02:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:02:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:02:58' 2026-02-22T20:02:59.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:02:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:02:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:02:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:02:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:02:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:02:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:02:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:02:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:02:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:02:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:02:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:03:01.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:03:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:03:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:03:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:03:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:03:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:03:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:03:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:03:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:03:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:03:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:03:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:03:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:03:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:03:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:03:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:03:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:03:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:03:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:03:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:03:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:03:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:03:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:03:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:03:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:03:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:03:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:03:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:03:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:04.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:04.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:03:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:03:04.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:04.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:03:04.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:03:04.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:03:04.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:03:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 116 2026-02-22T20:03:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:03:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:03:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:03:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:03:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:03:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:03:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:03:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:03:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:03:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:03:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:03:08.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=116 2026-02-22T20:03:08.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:03:08.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:03:08.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:03:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:03:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:03:08.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:03:08.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=116 2026-02-22T20:03:08.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 116 = 116 2026-02-22T20:03:08.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:08.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:03:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:03:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:09.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:09.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:03:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:03:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:03:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:08 2026-02-22T20:03:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:03:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:03:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790580,"replay_state":"idle"} 2026-02-22T20:03:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:04 2026-02-22T20:03:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:03:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 116 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.d55d29f2-71ac-4917-a68f-54c464ff9b98 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:03:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:03:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:03:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:03:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:03:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:03:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:08' 2026-02-22T20:03:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:03:08' 2026-02-22T20:03:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:08' 2026-02-22T20:03:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:09.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:03:08cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790580,"replay_state":"idle"}2026-02-22 20:03:04' 2026-02-22T20:03:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:09.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:08' 2026-02-22T20:03:09.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:03:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:03:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:03:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:03:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:03:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:03:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:03:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:03:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790580,"replay_state":"idle"} 2026-02-22T20:03:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:04 2026-02-22T20:03:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:03:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:03:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:03:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:08' 2026-02-22T20:03:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:03:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:03:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790580,"replay_state":"idle"} 2026-02-22T20:03:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:03:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:03:10.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:04' 2026-02-22T20:03:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:03:04' 2026-02-22T20:03:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:04' 2026-02-22T20:03:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:10.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790580,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:03:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:03:08' 2026-02-22T20:03:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:10.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:04' 2026-02-22T20:03:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:10.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:03:10.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:03:10.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:03:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:03:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:03:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:03:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:03:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:03:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:03:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:03:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:03:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:03:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:03:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:03:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:03:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:03:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:03:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:03:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:03:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:03:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:12.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:12.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:03:12.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:12.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:12.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:12.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:12.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:03:12.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:03:12.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:12.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:03:12.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:03:12.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:03:12.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:12.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:03:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:03:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:03:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:03:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:03:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:14.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:03:14.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:14.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:03:14.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:14.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:14.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:14.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:14.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:14.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:14.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:14.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:03:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:03:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:14 2026-02-22T20:03:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:03:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:13' 2026-02-22T20:03:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:03:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:03:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:03:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:15.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:15.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:14' 2026-02-22T20:03:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:03:14' 2026-02-22T20:03:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:14' 2026-02-22T20:03:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:03:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:03:13' 2026-02-22T20:03:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:14' 2026-02-22T20:03:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:15.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:03:15.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:15.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:03:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:03:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:03:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:13 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:14' 2026-02-22T20:03:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:03:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:03:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:03:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:16.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:13' 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:03:13' 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:13' 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:16.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:03:13cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:03:14' 2026-02-22T20:03:16.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:16.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:13' 2026-02-22T20:03:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:16.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:03:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:03:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:03:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:03:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:03:17.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:03:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:03:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:03:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:03:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:03:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:03:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:03:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:03:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:03:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:03:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:03:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:03:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:03:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:18.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:03:18.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:03:18.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:03:18.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:18.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:03:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 119 2026-02-22T20:03:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:03:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:03:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:22.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:03:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:03:22.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:03:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:03:22.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:03:22.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:03:22.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:22.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:22.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:22.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:22.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:22.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:03:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:03:22.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:03:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=119 2026-02-22T20:03:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:03:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:03:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:03:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:03:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:03:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=119 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 119 = 119 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:03:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:03:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:19 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790596,"replay_state":"idle"} 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:18 2026-02-22T20:03:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:03:24.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 119 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.8b48761e-7133-4024-b378-2f42e7e45457 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:03:24.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:03:24.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:03:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:03:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:03:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:03:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:19' 2026-02-22T20:03:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:03:19' 2026-02-22T20:03:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:19' 2026-02-22T20:03:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:03:19cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790596,"replay_state":"idle"}2026-02-22 20:03:18' 2026-02-22T20:03:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:24.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:19' 2026-02-22T20:03:24.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:24.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:03:24.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:03:24.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:03:24.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:03:24.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:24.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:24.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:03:24.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:03:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:03:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790596,"replay_state":"idle"} 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:18 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:24' 2026-02-22T20:03:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:03:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:03:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790596,"replay_state":"idle"} 2026-02-22T20:03:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:03:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:03:25.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:18' 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:03:18' 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:18' 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790596,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:03:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:03:24' 2026-02-22T20:03:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:25.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:18' 2026-02-22T20:03:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:25.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:03:25.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:03:25.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:03:25.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:03:25.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:25.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:25.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:03:25.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:03:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:03:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:03:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:03:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:03:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:03:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:03:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:03:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:03:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:03:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:03:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:03:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:03:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:03:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:03:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:03:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:26.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:26.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:03:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:03:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:03:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:03:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:03:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:03:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:03:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:03:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:03:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:03:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:03:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:03:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:28.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:28.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:28.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:03:28.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:03:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:03:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:03:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:03:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:03:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:03:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:03:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:28.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:29.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:03:29.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:29.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:29.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:03:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:29.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:29.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:29 2026-02-22T20:03:29.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:29.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:03:29.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:29.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:29.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:28' 2026-02-22T20:03:29.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:03:29.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:03:29.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:29.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:03:29.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:29.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:29.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:29.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:29.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:29.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:29.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:29.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:29' 2026-02-22T20:03:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:03:29' 2026-02-22T20:03:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:29' 2026-02-22T20:03:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:03:29cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:03:28' 2026-02-22T20:03:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:29' 2026-02-22T20:03:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:03:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:03:30.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:03:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:03:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:28 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:29' 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:03:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:03:31.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:31.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:03:31.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:31.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:31.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:31.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:31.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:31.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:31.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:28' 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:03:28' 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:28' 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:31.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:03:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:03:29' 2026-02-22T20:03:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:28' 2026-02-22T20:03:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:03:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:03:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:03:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:03:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:03:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:03:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:03:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:03:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:03:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:03:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:34.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:34.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:03:34.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:03:34.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:03:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:03:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:03:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:03:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:03:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:03:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:03:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:03:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:03:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:03:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:36.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:03:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:03:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:03:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 122 2026-02-22T20:03:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:03:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:03:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:03:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:03:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:03:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:03:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:03:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:03:38.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:38.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:38.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:38.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:38.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:38.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:38.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:03:38.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:03:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:03:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=122 2026-02-22T20:03:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:03:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:03:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:03:38.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:03:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:03:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:03:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=122 2026-02-22T20:03:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 122 = 122 2026-02-22T20:03:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:03:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:39.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:03:39.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:39.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:39.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:39.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:39.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:39.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:39.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:03:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:03:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:03:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:38 2026-02-22T20:03:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:03:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:03:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790612,"replay_state":"idle"} 2026-02-22T20:03:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:34 2026-02-22T20:03:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:03:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 121 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.51b48a9c-24d0-4699-831c-b2d96f3c22b5 (peer_uuids:[]) 2026-02-22T20:03:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 122 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.e972cb4e-0f3c-4aef-955c-fdde9f4157bc (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:03:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:03:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:03:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:03:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:03:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:39.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:03:39.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:39.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:39.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:39.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:39.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:38' 2026-02-22T20:03:39.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:03:38' 2026-02-22T20:03:39.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:39.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:39.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:39.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:39.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:38' 2026-02-22T20:03:39.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:39.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:39.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:39.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:03:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790612,"replay_state":"idle"}2026-02-22 20:03:34' 2026-02-22T20:03:39.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:39.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:38' 2026-02-22T20:03:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:39.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:03:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:03:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:03:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:03:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:03:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:39.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:39.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:39.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:03:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:03:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:03:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790612,"replay_state":"idle"} 2026-02-22T20:03:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:34 2026-02-22T20:03:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:03:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:03:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:03:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:38' 2026-02-22T20:03:40.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:03:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:03:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790612,"replay_state":"idle"} 2026-02-22T20:03:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:03:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:03:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:40.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:40.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:40.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:34' 2026-02-22T20:03:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:03:34' 2026-02-22T20:03:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:34' 2026-02-22T20:03:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790612,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:03:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:03:38' 2026-02-22T20:03:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:34' 2026-02-22T20:03:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:40.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:03:40.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:03:40.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:03:40.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:03:40.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:40.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:40.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:03:40.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:03:40.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:40.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:40.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:40.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:03:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:03:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:03:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:03:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:03:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:03:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:03:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:03:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:03:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:03:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:03:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:03:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:03:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:03:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:03:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:03:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:03:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:03:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:03:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:03:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:03:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:43.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:03:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:44 2026-02-22T20:03:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:03:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:43' 2026-02-22T20:03:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:03:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:03:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:03:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:44.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:44.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:44' 2026-02-22T20:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:03:44' 2026-02-22T20:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:44' 2026-02-22T20:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:03:44cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:03:43' 2026-02-22T20:03:44.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:44.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:44' 2026-02-22T20:03:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:03:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:03:44.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:44.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:03:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:03:45.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:45.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:45.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:45.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:45.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:43 2026-02-22T20:03:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:03:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:03:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:03:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:44' 2026-02-22T20:03:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:03:45.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:03:45.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:45.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:03:45.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:45.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:45.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:45.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:45.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:45.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:45.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:43' 2026-02-22T20:03:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:03:43' 2026-02-22T20:03:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:43' 2026-02-22T20:03:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:03:43cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:03:44' 2026-02-22T20:03:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:45.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:43' 2026-02-22T20:03:45.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:03:45.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:03:45.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:03:45.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:03:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:03:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:03:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:03:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:03:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:03:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:03:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:03:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:03:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:47.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:03:47.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:47.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:47.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:47.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:03:47.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:47.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:47.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:03:47.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:47.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:03:47.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:47.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:03:47.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:03:47.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:47.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:47.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:47.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:03:47.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:03:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:03:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:03:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:03:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:03:47.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:47.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:47.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:47.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:47.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:03:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:03:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:03:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:03:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:03:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:03:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:03:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:03:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:03:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:03:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:49.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:03:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:49.949 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:03:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:03:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:03:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:03:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:03:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:03:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:03:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:03:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:03:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:03:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:03:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 125 2026-02-22T20:03:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:03:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:03:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:53.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:53.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:03:53.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:03:53.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:03:53.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:53.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:03:53.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:03:53.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:03:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:03:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:03:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:03:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:03:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=125 2026-02-22T20:03:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:03:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:03:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:03:54.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:54.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:54.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:03:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:03:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:03:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:03:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=125 2026-02-22T20:03:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 125 = 125 2026-02-22T20:03:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:03:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:03:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:03:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:54 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790625,"replay_state":"idle"} 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:48 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 125 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.cbcf56dd-cc28-4854-935a-6f385d4977bd (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:03:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:03:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:03:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:03:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:03:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:03:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:55.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:54' 2026-02-22T20:03:55.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:03:54' 2026-02-22T20:03:55.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:03:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:54' 2026-02-22T20:03:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:03:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790625,"replay_state":"idle"}2026-02-22 20:03:48' 2026-02-22T20:03:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:54' 2026-02-22T20:03:55.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:03:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:03:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:03:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:03:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:03:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:03:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:03:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:03:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:03:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790625,"replay_state":"idle"} 2026-02-22T20:03:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:03:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:48 2026-02-22T20:03:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:03:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:03:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:03:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:03:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:54' 2026-02-22T20:03:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:03:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:03:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:03:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790625,"replay_state":"idle"} 2026-02-22T20:03:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:03:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:03:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:03:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:03:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:03:56.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:03:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:03:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:03:56.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:48' 2026-02-22T20:03:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:03:48' 2026-02-22T20:03:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:03:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:48' 2026-02-22T20:03:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:03:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:03:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:03:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790625,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:03:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:03:54' 2026-02-22T20:03:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:03:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:03:56.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:48' 2026-02-22T20:03:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:03:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:03:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:03:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:03:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:03:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:03:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:03:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:03:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:03:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:03:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:03:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:03:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:03:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:03:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:03:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:57.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:03:57.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:03:57.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:57.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:03:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:03:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:03:57.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:03:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:03:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:03:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:03:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:03:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:03:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:03:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:03:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:03:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:03:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:03:59.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:03:59.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:03:59.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:59.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:59.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:03:59.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:03:59.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:03:59.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:03:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:03:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:03:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:03:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:03:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:03:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:03:59.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:03:59.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:03:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:03:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:03:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:03:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:03:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:03:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:03:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:03:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:03:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:03:59.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:59.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:59.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:03:59.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:03:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:03:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:03:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:03:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:03:59.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:03:59.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:03:59.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:03:59.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:03:59.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:03:59.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:03:59.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:03:59.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:03:59.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:03:59.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:03:59.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:04:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:04:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:59 2026-02-22T20:04:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:04:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:04:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:58' 2026-02-22T20:04:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:04:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:04:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:04:01.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:01.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:01.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:01.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:01.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:01.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:01.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:59' 2026-02-22T20:04:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:03:59' 2026-02-22T20:04:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:59' 2026-02-22T20:04:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:01.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:03:59cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 20:03:58' 2026-02-22T20:04:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:59' 2026-02-22T20:04:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:01.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:04:01.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:01.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:04:01.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:04:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:02.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:04:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:04:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:04:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:58 2026-02-22T20:04:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:04:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:03:59' 2026-02-22T20:04:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:04:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:04:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:04:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:58' 2026-02-22T20:04:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:03:58' 2026-02-22T20:04:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:03:58' 2026-02-22T20:04:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted44434452mirror.0trial0982026-02-22 20:03:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:03:59' 2026-02-22T20:04:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:03:58' 2026-02-22T20:04:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:04:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:04:02.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:04:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:02.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:04:02.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:04:03.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:04:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:04:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:04:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:04:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:04:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:04:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:04:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:04:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:04:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:04:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:04:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:04:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:04:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:04:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:04.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:04:04.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:04:04.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:04.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:04:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:04:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:04:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 128 2026-02-22T20:04:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:04:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:04:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:04:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:04:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:04:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:04:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:04:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:04:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:04:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:04:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:04:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=128 2026-02-22T20:04:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:04:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:04:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:04:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:08.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:08.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:08.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:08.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:04:08.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:04:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:04:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=128 2026-02-22T20:04:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 128 = 128 2026-02-22T20:04:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:04:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:09.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:04:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:04:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:04:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:04:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:08 2026-02-22T20:04:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:04:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:04:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790642,"replay_state":"idle"} 2026-02-22T20:04:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:04 2026-02-22T20:04:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:04:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 128 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.7761b820-41c0-41c5-bb8a-9351dced5bb7 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:04:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:04:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:04:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:04:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:04:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:04:09.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:09.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:08' 2026-02-22T20:04:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:04:08' 2026-02-22T20:04:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:08' 2026-02-22T20:04:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:09.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:04:08cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790642,"replay_state":"idle"}2026-02-22 20:04:04' 2026-02-22T20:04:09.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:09.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:08' 2026-02-22T20:04:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:04:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:04:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:04:09.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:04:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:04:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:10.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:04:10.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:10.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:10.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:10.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:10.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:10.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:04:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:04:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790642,"replay_state":"idle"} 2026-02-22T20:04:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:04 2026-02-22T20:04:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:04:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:04:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:04:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:08' 2026-02-22T20:04:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:04:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:04:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790642,"replay_state":"idle"} 2026-02-22T20:04:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:04:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:04:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:04' 2026-02-22T20:04:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:04:04' 2026-02-22T20:04:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:04' 2026-02-22T20:04:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:10.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790642,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:04:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:04:08' 2026-02-22T20:04:10.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:10.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:04' 2026-02-22T20:04:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:10.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:04:10.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:04:10.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:04:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:04:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:04:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:04:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:04:11.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:04:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:04:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:04:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:11.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:04:11.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:04:11.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:11.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:04:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:04:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:04:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:04:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:04:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:04:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:04:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:04:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:04:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:04:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:04:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:12.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:04:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:04:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:04:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:04:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:04:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:04:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:04:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:14.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:04:14.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:14.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:14.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:04:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:04:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:04:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:04:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:04:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:04:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:04:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:14 2026-02-22T20:04:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:04:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:13' 2026-02-22T20:04:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:04:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:04:15.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:15.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:04:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:15.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:14' 2026-02-22T20:04:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:04:14' 2026-02-22T20:04:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:14' 2026-02-22T20:04:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:04:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:04:13' 2026-02-22T20:04:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:14' 2026-02-22T20:04:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:04:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:04:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:04:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:04:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:13 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:14' 2026-02-22T20:04:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:04:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:04:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:04:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:13' 2026-02-22T20:04:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:04:13' 2026-02-22T20:04:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:13' 2026-02-22T20:04:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:16.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:04:13cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:04:14' 2026-02-22T20:04:16.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:16.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:13' 2026-02-22T20:04:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:16.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:04:16.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:16.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:04:16.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:04:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:04:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:04:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:04:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:04:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:04:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:04:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:04:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:04:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:04:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:04:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:04:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:04:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:04:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 131 2026-02-22T20:04:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:04:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:04:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:04:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:04:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:04:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:22.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:04:22.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:04:22.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:04:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:04:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:04:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:04:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=131 2026-02-22T20:04:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:04:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:04:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:04:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:04:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:04:23.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:04:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=131 2026-02-22T20:04:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 131 = 131 2026-02-22T20:04:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:04:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:04:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:04:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:04:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:04:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:19 2026-02-22T20:04:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:04:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:04:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790656,"replay_state":"idle"} 2026-02-22T20:04:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:18 2026-02-22T20:04:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:04:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 131 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.5256e09c-7f1f-41ff-9c1b-d126484466ea (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:04:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:04:24.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:04:24.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:04:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:04:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:04:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:19' 2026-02-22T20:04:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:04:19' 2026-02-22T20:04:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:19' 2026-02-22T20:04:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:04:19cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790656,"replay_state":"idle"}2026-02-22 20:04:18' 2026-02-22T20:04:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:24.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:19' 2026-02-22T20:04:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:24.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:04:24.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:04:24.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:04:24.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:04:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:04:25.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790656,"replay_state":"idle"} 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:18 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:24' 2026-02-22T20:04:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:04:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:04:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:25.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790656,"replay_state":"idle"} 2026-02-22T20:04:25.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:25.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:04:25.195 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:04:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:18' 2026-02-22T20:04:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:04:18' 2026-02-22T20:04:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:18' 2026-02-22T20:04:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790656,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:04:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:04:24' 2026-02-22T20:04:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:18' 2026-02-22T20:04:25.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:04:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:04:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:04:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:04:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:04:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:04:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:04:25.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:04:25.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:04:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:04:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:04:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:04:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:04:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:04:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:04:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:04:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:04:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:04:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:04:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:04:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:04:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:04:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:04:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:04:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:04:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:04:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:04:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:04:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:04:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:04:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:04:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:04:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:04:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:04:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:04:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:04:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:04:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:29 2026-02-22T20:04:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:04:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:28' 2026-02-22T20:04:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:04:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:04:30.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:30.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:04:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:29' 2026-02-22T20:04:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:04:29' 2026-02-22T20:04:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:29' 2026-02-22T20:04:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:04:29cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:04:28' 2026-02-22T20:04:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:30.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:29' 2026-02-22T20:04:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:04:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:04:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:04:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:04:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:31.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:31.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:04:31.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:31.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:31.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:31.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:31.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:31.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:28 2026-02-22T20:04:31.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:31.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:04:31.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:31.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:31.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:29' 2026-02-22T20:04:31.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:04:31.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:04:31.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:31.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:04:31.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:31.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:31.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:31.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:31.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:31.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:28' 2026-02-22T20:04:31.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:04:28' 2026-02-22T20:04:31.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:31.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:31.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:31.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:31.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:28' 2026-02-22T20:04:31.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:31.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:31.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:31.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:04:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:04:29' 2026-02-22T20:04:31.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:31.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:31.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:28' 2026-02-22T20:04:31.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:04:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:31.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:04:31.154 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:04:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:04:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:04:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:04:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:04:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:04:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:04:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:04:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:33.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:04:33.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:33.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:33.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:33.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:04:33.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:33.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:33.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:04:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:04:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:04:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:04:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:04:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:04:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:04:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:04:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:04:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:04:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:04:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:04:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:04:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:04:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:04:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:04:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:04:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:04:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:04:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:04:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:04:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:04:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:04:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:04:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:35.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:35.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:35.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:04:35.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:04:35.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:35.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:04:35.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:04:35.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:04:35.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:35.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:04:35.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:04:35.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:04:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 134 2026-02-22T20:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:04:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:04:39.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:04:39.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:04:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:04:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:04:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:04:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=134 2026-02-22T20:04:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:04:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:04:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:04:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:04:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:04:39.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:04:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=134 2026-02-22T20:04:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 134 = 134 2026-02-22T20:04:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:04:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:04:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:04:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:04:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:04:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:38 2026-02-22T20:04:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:04:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:04:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790671,"replay_state":"idle"} 2026-02-22T20:04:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:34 2026-02-22T20:04:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:04:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 134 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.ae8d0c4d-9201-4c75-8bc5-1d2bc0265655 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:04:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:04:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:04:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:04:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:04:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:04:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:38' 2026-02-22T20:04:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:04:38' 2026-02-22T20:04:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:38' 2026-02-22T20:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:04:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790671,"replay_state":"idle"}2026-02-22 20:04:34' 2026-02-22T20:04:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:40.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:38' 2026-02-22T20:04:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:40.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:04:40.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:04:40.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:04:40.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:04:40.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:41.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:04:41.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:41.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:41.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:41.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:41.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:41.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:41.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:41.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:41.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:04:41.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:41.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:41.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:04:41.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790671,"replay_state":"idle"} 2026-02-22T20:04:41.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:41.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:34 2026-02-22T20:04:41.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:41.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:04:41.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:04:41.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:04:41.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:38' 2026-02-22T20:04:41.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:04:41.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:04:41.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:41.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790671,"replay_state":"idle"} 2026-02-22T20:04:41.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:41.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:04:41.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:41.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:04:41.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:41.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:41.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:41.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:34' 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:04:34' 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:34' 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:41.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790671,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:04:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:04:38' 2026-02-22T20:04:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:41.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:34' 2026-02-22T20:04:41.718 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:04:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:04:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:04:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:04:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:04:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:04:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:04:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:04:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:04:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:04:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:04:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:04:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:04:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:04:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:04:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:04:43.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:04:43.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:43.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:04:43.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:04:43.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:04:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:04:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:04:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:04:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:04:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:04:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:04:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:04:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:04:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:04:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:46.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:04:46.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:44 2026-02-22T20:04:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:04:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:43' 2026-02-22T20:04:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:04:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:04:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:04:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:44' 2026-02-22T20:04:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:04:44' 2026-02-22T20:04:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:44' 2026-02-22T20:04:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:04:44cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:04:43' 2026-02-22T20:04:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:44' 2026-02-22T20:04:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:04:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:04:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:04:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:04:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:43 2026-02-22T20:04:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:04:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:04:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:04:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:44' 2026-02-22T20:04:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:04:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:04:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:04:47.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:47.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:47.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:43' 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:04:43' 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:43' 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:04:43cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:04:44' 2026-02-22T20:04:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:43' 2026-02-22T20:04:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:47.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:04:47.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:04:47.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:04:47.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:04:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:04:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:04:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:04:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:04:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:04:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:04:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:04:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:04:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:04:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:04:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:04:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:49.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:49.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:04:49.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:04:49.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:49.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:04:49.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:04:49.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:04:49.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:49.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:04:53.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 137 2026-02-22T20:04:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:04:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:04:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:04:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:04:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:04:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:04:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:04:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:04:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:04:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:04:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:04:53.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:04:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=137 2026-02-22T20:04:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:04:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:04:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:04:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:04:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:04:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:04:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:04:54.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=137 2026-02-22T20:04:54.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 137 = 137 2026-02-22T20:04:54.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:04:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:04:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:04:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:04:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:04:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:54 2026-02-22T20:04:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:04:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:04:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790687,"replay_state":"idle"} 2026-02-22T20:04:55.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:48 2026-02-22T20:04:55.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:04:55.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 137 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.9c3b2fce-be7d-4bed-ad86-93c4fc2b5639 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:04:55.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:04:55.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:04:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:04:55.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:55.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:04:55.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:04:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:54' 2026-02-22T20:04:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:04:54' 2026-02-22T20:04:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:04:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:54' 2026-02-22T20:04:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:55.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:04:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790687,"replay_state":"idle"}2026-02-22 20:04:48' 2026-02-22T20:04:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:54' 2026-02-22T20:04:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:04:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:04:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:04:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:04:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:04:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:56.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:04:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:04:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:04:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:04:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:04:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790687,"replay_state":"idle"} 2026-02-22T20:04:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:04:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:48 2026-02-22T20:04:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:04:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:04:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:04:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:04:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:54' 2026-02-22T20:04:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:04:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:04:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:04:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790687,"replay_state":"idle"} 2026-02-22T20:04:56.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:04:56.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:04:56.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:04:56.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:04:56.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:04:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:04:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:04:56.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:48' 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:04:48' 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:48' 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:04:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:04:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790687,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:04:48cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:04:54' 2026-02-22T20:04:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:04:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:04:56.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:48' 2026-02-22T20:04:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:04:56.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:04:56.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:04:56.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:04:56.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:04:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:04:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:04:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:04:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:04:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:04:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:04:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:04:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:04:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:04:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:04:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:04:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:04:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:04:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:04:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:04:57.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:57.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:57.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:57.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:57.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:04:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:04:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:57.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:04:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:04:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:04:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:57.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:04:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:04:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:04:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:04:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:04:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:04:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:04:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:04:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:04:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:04:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:04:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:04:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:04:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:04:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:04:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:04:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:04:59.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:04:59.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:04:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:04:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:04:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:04:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:04:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:04:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:04:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:04:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:05:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:05:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:59 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:58' 2026-02-22T20:05:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:05:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:05:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:05:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:01.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:59' 2026-02-22T20:05:01.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:04:59' 2026-02-22T20:05:01.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:01.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:01.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:01.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:01.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:59' 2026-02-22T20:05:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:04:59cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image demoted2026-02-22 20:04:58' 2026-02-22T20:05:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:01.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:59' 2026-02-22T20:05:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:05:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:05:01.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:05:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:05:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:58 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:05:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:04:59' 2026-02-22T20:05:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:05:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:05:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:05:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:58' 2026-02-22T20:05:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:04:58' 2026-02-22T20:05:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:04:58' 2026-02-22T20:05:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted44434452mirror.0trial0982026-02-22 20:04:58cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:04:59' 2026-02-22T20:05:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:04:58' 2026-02-22T20:05:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:05:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:05:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:05:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:05:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:05:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:05:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:05:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:05:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:05:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:05:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:05:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:05:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:05:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:05:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:05:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:05:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:05:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:05:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:05:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:05:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:05:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:05:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:05:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:05:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:05:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:05:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:05:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:05:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:05:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:05:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:05:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:05:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:05:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:05:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:05:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:05:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:05:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:05:06.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:06.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:06.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:06.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:05:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:05:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:05:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:05:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:05:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:05:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:05:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:05:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:05:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:05:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:05:10.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 140 2026-02-22T20:05:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:05:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:05:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:05:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:05:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:05:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:05:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:05:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:05:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:05:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:05:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:05:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:05:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:05:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:05:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:05:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=140 2026-02-22T20:05:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:05:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:05:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:05:10.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:05:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:05:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:05:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:05:10.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:05:10.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=140 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 140 = 140 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:05:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:05:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:05:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:05:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:08 2026-02-22T20:05:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:11.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:05:11.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:05:11.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790702,"replay_state":"idle"} 2026-02-22T20:05:11.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:04 2026-02-22T20:05:11.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:05:11.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 140 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.b77bb3fe-67f4-4aa0-8bca-525a646a38df (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:05:11.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:05:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:05:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:05:11.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:11.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:05:11.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:11.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:05:11.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:11.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:11.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:11.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:08' 2026-02-22T20:05:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:05:08' 2026-02-22T20:05:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:08' 2026-02-22T20:05:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:05:08cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790702,"replay_state":"idle"}2026-02-22 20:05:04' 2026-02-22T20:05:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:08' 2026-02-22T20:05:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:11.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:05:11.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:05:11.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:05:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:05:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:05:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:05:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:05:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:05:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790702,"replay_state":"idle"} 2026-02-22T20:05:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:04 2026-02-22T20:05:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:05:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:05:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:05:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:08' 2026-02-22T20:05:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:05:12.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:05:12.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:12.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790702,"replay_state":"idle"} 2026-02-22T20:05:12.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:12.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:05:12.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:12.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:05:12.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:04' 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:05:04' 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:04' 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790702,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:05:04cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:05:08' 2026-02-22T20:05:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:04' 2026-02-22T20:05:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:12.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:05:12.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:05:12.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:05:12.718 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:05:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:05:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test 2026-02-22T20:05:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T20:05:13.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:05:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:05:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:05:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:05:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:05:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:05:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:14.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:14.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:14.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:05:14.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:14.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:05:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:05:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:05:14.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:14.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:14.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:05:14.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:05:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:05:14.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:05:14.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:05:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:05:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:05:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:05:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:05:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:05:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:05:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:05:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:05:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:05:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:05:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:16.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:05:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:05:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:05:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:05:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:05:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:16.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:05:16.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:05:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:05:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:05:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:14 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:13' 2026-02-22T20:05:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:05:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:05:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:17.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:05:17.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:17.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:17.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:14' 2026-02-22T20:05:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:05:14' 2026-02-22T20:05:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:14' 2026-02-22T20:05:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:17.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image demoted43244333mirror.0trial0982026-02-22 20:05:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:05:13' 2026-02-22T20:05:17.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:17.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:14' 2026-02-22T20:05:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:05:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:05:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:05:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:05:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:05:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:05:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:18 2026-02-22T20:05:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:05:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:05:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:14' 2026-02-22T20:05:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:05:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:05:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:05:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:18' 2026-02-22T20:05:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:05:18' 2026-02-22T20:05:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:18' 2026-02-22T20:05:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:05:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 20:05:14' 2026-02-22T20:05:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:18.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:18' 2026-02-22T20:05:18.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:18.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:05:18.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:18.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:05:18.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test 2026-02-22T20:05:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:05:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T20:05:19.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:05:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T20:05:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:19.807 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T20:05:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:05:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:05:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:05:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:05:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:05:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:05:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:05:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:05:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:05:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:05:20.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:20.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:20.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:05:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:05:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:05:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:05:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:05:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:05:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:05:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:05:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:05:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:05:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:05:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:05:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:05:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:05:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:05:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:05:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:05:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:05:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:05:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T20:05:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:05:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:05:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:05:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:05:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:05:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:05:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:05:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:05:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:05:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:05:26.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:05:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:05:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:05:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:05:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:05:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:27.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:27.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T20:05:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 143 2026-02-22T20:05:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T20:05:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster2 2026-02-22T20:05:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:05:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:05:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:05:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:05:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:05:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:05:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:05:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:05:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:05:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:05:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:05:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:05:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:05:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=143 2026-02-22T20:05:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:05:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:05:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:05:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:05:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:05:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:05:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:05:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:05:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=143 2026-02-22T20:05:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 143 = 143 2026-02-22T20:05:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T20:05:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T20:05:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:05:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:29 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790718,"replay_state":"idle"} 2026-02-22T20:05:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:23 2026-02-22T20:05:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:05:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 143 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.fe0fab7b-dcc0-4108-b897-31cb52cfc0cb (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:05:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:05:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:05:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:05:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:05:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:05:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:29' 2026-02-22T20:05:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 20:05:29' 2026-02-22T20:05:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:29' 2026-02-22T20:05:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary43244333mirror.0trial0982026-02-22 20:05:29cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790718,"replay_state":"idle"}2026-02-22 20:05:23' 2026-02-22T20:05:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:29' 2026-02-22T20:05:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:05:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:05:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:05:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:05:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T20:05:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T20:05:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:33.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:33.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:33.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:33.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:33.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:33.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:05:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:05:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790718,"replay_state":"idle"} 2026-02-22T20:05:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:23 2026-02-22T20:05:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:05:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:05:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:05:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:29' 2026-02-22T20:05:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:05:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:05:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790718,"replay_state":"idle"} 2026-02-22T20:05:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:05:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:33.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:05:33.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:23' 2026-02-22T20:05:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 20:05:23' 2026-02-22T20:05:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:23' 2026-02-22T20:05:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790718,"replay_state":"idle"}44434452mirror.0trial0982026-02-22 20:05:23cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:05:29' 2026-02-22T20:05:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:33.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:23' 2026-02-22T20:05:33.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:33.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:05:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:05:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:05:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:05:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror test 2026-02-22T20:05:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T20:05:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:05:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:05:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:05:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:05:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:05:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:05:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:05:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:05:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:05:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:05:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:05:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:34.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:05:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:05:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:05:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:05:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = started 2026-02-22T20:05:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:05:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T20:05:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:05:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T20:05:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:05:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:05:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:05:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:05:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:05:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:05:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:05:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:05:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:05:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T20:05:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T20:05:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:05:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:05:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:34 2026-02-22T20:05:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:05:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:05:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:35' 2026-02-22T20:05:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:05:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:05:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:05:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:34' 2026-02-22T20:05:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 20:05:34' 2026-02-22T20:05:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:34' 2026-02-22T20:05:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary43244333mirror.0trial0982026-02-22 20:05:34cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:05:38' 2026-02-22T20:05:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:34' 2026-02-22T20:05:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:38.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:05:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:05:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T20:05:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T20:05:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:05:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:05:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:38 2026-02-22T20:05:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:05:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:05:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:05:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:34' 2026-02-22T20:05:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:05:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:05:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:05:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:38' 2026-02-22T20:05:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 20:05:38' 2026-02-22T20:05:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:05:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:38' 2026-02-22T20:05:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+unknownremote image is not primary44434452mirror.0trial0982026-02-22 20:05:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:05:34' 2026-02-22T20:05:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:39.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:38' 2026-02-22T20:05:39.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:05:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:05:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:05:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:05:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster2 mirror test 2026-02-22T20:05:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:05:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T20:05:40.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:05:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:05:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:05:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:05:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:05:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:05:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:05:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:05:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:05:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:05:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:05:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:05:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:05:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:05:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:05:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:05:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:05:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:05:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T20:05:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:05:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:41.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:05:41.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:05:41.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T20:05:41.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:41.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:41.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:41.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T20:05:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 146 2026-02-22T20:05:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T20:05:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:05:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:45.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:45.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:45.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T20:05:45.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.status 2026-02-22T20:05:45.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:05:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:05:45.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T20:05:45.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.status 2026-02-22T20:05:45.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:05:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:05:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:05:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.status 2026-02-22T20:05:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T20:05:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:05:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:05:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=146 2026-02-22T20:05:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:05:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:05:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:05:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:05:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.status 2026-02-22T20:05:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T20:05:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:05:45.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:05:45.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=146 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 146 = 146 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T20:05:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T20:05:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:43 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790739,"replay_state":"idle"} 2026-02-22T20:05:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:39 2026-02-22T20:05:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:05:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 146 .mirror.primary.3bad19be-bf49-42ca-8b9d-5d734494ef44.0a3e27cd-35fd-4e41-bc6a-b2707dd6b5f8 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:05:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:05:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:05:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:05:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:05:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:05:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:43' 2026-02-22T20:05:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 20:05:43' 2026-02-22T20:05:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:43' 2026-02-22T20:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:46.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:05:43cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790739,"replay_state":"idle"}2026-02-22 20:05:39' 2026-02-22T20:05:46.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:46.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:46.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:43' 2026-02-22T20:05:46.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:46.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:05:46.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:05:46.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:05:46.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:05:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:05:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:47.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:05:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:05:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:05:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 3bad19be-bf49-42ca-8b9d-5d734494ef44 2026-02-22T20:05:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:05:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790739,"replay_state":"idle"} 2026-02-22T20:05:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:05:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:39 2026-02-22T20:05:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:05:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:05:47.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:05:47.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:43' 2026-02-22T20:05:47.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:05:47.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:05:47.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:47.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790739,"replay_state":"idle"} 2026-02-22T20:05:47.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:47.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:05:47.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:05:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:47.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:05:47.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:47.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:39' 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:05:39' 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:39' 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:47.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:47.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790739,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:05:39cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:05:43' 2026-02-22T20:05:47.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:47.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:05:47.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:39' 2026-02-22T20:05:47.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:05:47.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:05:47.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:05:47.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:05:47.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_mirror_snaps cluster1 mirror test 2026-02-22T20:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:05:47.690 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:05:47.690 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 snap ls mirror/test --all 2026-02-22T20:05:47.690 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c -F ' mirror (' 2026-02-22T20:05:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 3 -le 3 2026-02-22T20:05:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_mirror_snaps cluster2 mirror test 2026-02-22T20:05:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:05:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:05:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:05:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 snap ls mirror/test --all 2026-02-22T20:05:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c -F ' mirror (' 2026-02-22T20:05:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 1 -le 3 2026-02-22T20:05:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: force promote' 2026-02-22T20:05:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:05:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:05:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:05:47 TEST: force promote 2026-02-22T20:05:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:05:47 TEST: force promote 2026-02-22T20:05:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ force_promote_image=test_force_promote 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote snapshot 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:05:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test_force_promote 2026-02-22T20:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test_force_promote 2026-02-22T20:05:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:05:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:05:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_force_promote snapshot 2026-02-22T20:05:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:05:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_force_promote snapshot 2026-02-22T20:05:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:05:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_force_promote 2026-02-22T20:05:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_force_promote': 2026-02-22T20:05:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:05:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:05:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 1503be1e2bdc 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.1503be1e2bdc 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:05:47 2026 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:05:47 2026 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:05:47 2026 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:05:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:05:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: c46ca8be-4aa3-4d70-ad27-57de82d3a393 2026-02-22T20:05:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-02-22T20:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:05:48.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:05:48.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:05:48.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:05:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:05:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 6250.21 bytes/sec: 24 MiB/s 2026-02-22T20:05:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-02-22T20:05:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T20:05:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:05:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:05:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T20:05:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:05:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:05:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:05:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:05:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:49.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:49.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:49.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:05:49.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:05:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:05:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:05:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:49.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:49.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:49.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:05:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T20:05:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:05:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:49.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:05:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:05:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:05:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:05:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:05:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:05:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:05:49.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T20:05:49.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:05:49.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:2 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:2 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:2 2026-02-22T20:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:05:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T20:05:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:05:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:05:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:05:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:05:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:05:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:05:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:05:49.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T20:05:49.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:05:49.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:05:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:3 2026-02-22T20:05:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:05:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:3 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:3 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:05:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:05:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T20:05:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:05:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:05:49.574 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:05:49.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:05:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T20:05:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:05:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:05:49.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:49.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T20:05:50.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:05:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:05:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:05:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:05:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T20:05:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:05:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:05:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:05:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:05:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:05:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:05:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:05:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:05:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:05:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:05:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:05:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:05:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:05:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:05:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:05:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:05:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote 2026-02-22T20:05:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote started 2026-02-22T20:05:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:50.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:50.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:05:50.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:05:50.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:50.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:51.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_force_promote started 2026-02-22T20:05:51.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:51.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:51.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:51.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:05:51.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:05:51.772 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:05:51.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:51.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:05:51.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:05:51.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:51.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:05:51.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:05:51.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:05:51.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:51.774 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:05:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:05:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:05:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:05:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:51.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:05:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:05:51.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:05:51.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:05:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:05:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:05:51.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T20:05:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:05:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:05:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:05:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote 2026-02-22T20:05:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:05:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:05:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:05:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:05:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:05:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:05:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:05:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:05:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:05:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:05:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:05:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:05:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:05:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:05:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:05:51.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:05:51.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:05:51.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:05:51.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:05:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:05:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_force_promote 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote 2026-02-22T20:05:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote 2026-02-22T20:05:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 148 2026-02-22T20:05:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_force_promote 2026-02-22T20:05:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:05:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_force_promote.status 2026-02-22T20:05:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_force_promote.status 2026-02-22T20:05:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:05:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_force_promote.status 2026-02-22T20:05:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_force_promote.status 2026-02-22T20:05:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_force_promote.status 2026-02-22T20:05:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:05:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote.status 2026-02-22T20:05:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_force_promote /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_force_promote.status 2026-02-22T20:05:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_force_promote.status 2026-02-22T20:05:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_force_promote --format xml 2026-02-22T20:05:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:05:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:05:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=148 2026-02-22T20:05:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:05:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:05:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:05:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote.status 2026-02-22T20:05:52.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:52.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:52.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:52.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote.status 2026-02-22T20:05:52.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote --format xml 2026-02-22T20:05:52.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:05:52.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:05:52.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=147 2026-02-22T20:05:52.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 148 = 147 2026-02-22T20:05:52.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:05:52.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:05:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote.status 2026-02-22T20:05:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote.status 2026-02-22T20:05:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote --format xml 2026-02-22T20:05:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:05:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:05:52.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=148 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 148 = 148 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+replaying 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:53.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+replaying '' '' 2026-02-22T20:05:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:53.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-02-22T20:05:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_force_promote: 2026-02-22T20:05:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: c46ca8be-4aa3-4d70-ad27-57de82d3a393 2026-02-22T20:05:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:05:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790748,"replay_state":"idle"} 2026-02-22T20:05:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:05:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:50 2026-02-22T20:05:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:05:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:05:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:05:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:53' 2026-02-22T20:05:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:05:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:05:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790748,"replay_state":"idle"} 2026-02-22T20:05:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:05:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:05:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:05:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:50' 2026-02-22T20:05:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+replaying '2026-02-22 20:05:50' 2026-02-22T20:05:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:05:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:50' 2026-02-22T20:05:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790739,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:05:39cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:05:53test_force_promotec46ca8be-4aa3-4d70-ad27-57de82d3a393up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790748,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 20:05:50cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:05:53' 2026-02-22T20:05:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T20:05:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:50' 2026-02-22T20:05:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T20:05:54.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:05:54.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:05:54.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:05:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-02-22T20:05:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:05:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:05:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_force_promote: 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: c46ca8be-4aa3-4d70-ad27-57de82d3a393 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:53 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":136533.33,"bytes_per_snapshot":204800.0,"last_snapshot_bytes":409600,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790752,"remote_snapshot_timestamp":1771790752,"replay_state":"idle"} 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:54 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 147 .mirror.primary.c46ca8be-4aa3-4d70-ad27-57de82d3a393.0367d9f0-0759-4216-8e1d-415f5b725430 (peer_uuids:[]) 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 148 .mirror.primary.c46ca8be-4aa3-4d70-ad27-57de82d3a393.cc575d20-c2d3-43d6-935c-69f2728eaa49 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:05:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:05:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:05:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:05:55.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:05:55.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:05:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:05:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:05:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:05:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:05:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:05:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:05:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:53' 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-02-22 20:05:53' 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:53' 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:05:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:05:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK244434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:05:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:05:54test_force_promotec46ca8be-4aa3-4d70-ad27-57de82d3a393up+stoppedlocal image is primary44474457mirror.1trial0982026-02-22 20:05:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":136533.33,"bytes_per_snapshot":204800.0,"last_snapshot_bytes":409600,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790752,"remote_snapshot_timestamp":1771790752,"replay_state":"idle"}2026-02-22 20:05:54' 2026-02-22T20:05:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:05:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T20:05:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:53' 2026-02-22T20:05:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T20:05:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:05:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:05:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:05:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:05:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:05:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test_force_promote --force 2026-02-22T20:05:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force=--force 2026-02-22T20:05:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote --force 2026-02-22T20:05:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:05:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote 2026-02-22T20:05:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote stopped 2026-02-22T20:05:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:05:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:05:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:05:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:05:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:05:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:05:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_force_promote stopped 2026-02-22T20:06:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:06:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:06:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:06:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-02-22T20:06:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:06:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:06:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:06:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:06:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:06:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:06:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-02-22T20:06:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:00.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:06:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:06:00.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T20:06:00.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:00.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote 2026-02-22T20:06:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:06:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:06:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:06:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:06:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:06:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:00.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:00.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:00.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:06:00.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:06:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:06:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:06:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:06:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:06:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T20:06:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:06:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:06:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:06:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-02-22T20:06:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:06:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:06:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:06:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:06:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:06:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:06:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:06:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-22T20:06:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:06:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:06:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:06:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:06:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:06:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:06:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:06:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:06:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T20:06:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-22T20:06:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:06:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:06:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:06:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:06:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:06:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:06:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:06:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:06:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:06:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T20:06:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:06:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:06:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:06:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:06:02.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:06:02.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:02.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+stopped 2026-02-22T20:06:02.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:02.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:06:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+stopped '' '' 2026-02-22T20:06:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:06:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-02-22T20:06:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_force_promote: 2026-02-22T20:06:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: c46ca8be-4aa3-4d70-ad27-57de82d3a393 2026-02-22T20:06:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:06:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:06:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:06:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:59 2026-02-22T20:06:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:06:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:06:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:06:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:06:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:06:03 2026-02-22T20:06:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:06:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 150 .mirror.primary.c46ca8be-4aa3-4d70-ad27-57de82d3a393.2d647bd4-0f0d-4cda-a62e-b7d7aa910727 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:06:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:06:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:06:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:06:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:06:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:06:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:06:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:06:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:06:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:06:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:06:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:06:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:06:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:59' 2026-02-22T20:06:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+stopped '2026-02-22 20:05:59' 2026-02-22T20:06:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:06:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:05:59' 2026-02-22T20:06:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:06:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK1143244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:05:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:06:03test_force_promotec46ca8be-4aa3-4d70-ad27-57de82d3a393up+stoppedlocal image is primary43284339mirror.1trial0982026-02-22 20:05:59cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:06:03' 2026-02-22T20:06:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:06:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T20:06:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:05:59' 2026-02-22T20:06:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T20:06:03.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:06:03.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:06:03.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:06:03.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:06:03.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-02-22T20:06:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:06:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-02-22T20:06:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_force_promote: 2026-02-22T20:06:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: c46ca8be-4aa3-4d70-ad27-57de82d3a393 2026-02-22T20:06:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:06:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:06:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:06:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:06:03 2026-02-22T20:06:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:06:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:06:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:06:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:06:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:05:59 2026-02-22T20:06:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:06:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 147 .mirror.primary.c46ca8be-4aa3-4d70-ad27-57de82d3a393.0367d9f0-0759-4216-8e1d-415f5b725430 (peer_uuids:[]) 2026-02-22T20:06:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 148 .mirror.primary.c46ca8be-4aa3-4d70-ad27-57de82d3a393.cc575d20-c2d3-43d6-935c-69f2728eaa49 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:06:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:06:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:06:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:06:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:06:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:06:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:06:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:06:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:06:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:06:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:06:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:06:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:06:03' 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-02-22 20:06:03' 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:06:03' 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:06:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK1144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:06:03cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:05:54test_force_promotec46ca8be-4aa3-4d70-ad27-57de82d3a393up+stoppedlocal image is primary44474457mirror.1trial0982026-02-22 20:06:03cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:05:59' 2026-02-22T20:06:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:06:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T20:06:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:06:03' 2026-02-22T20:06:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T20:06:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:06:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:06:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:06:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:06:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster1 mirror test_force_promote 100 2026-02-22T20:06:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:06:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:06:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:06:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:06:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:06:04.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:06:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:06:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.61 bytes/sec: 33 MiB/s 2026-02-22T20:06:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-02-22T20:06:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:06:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:06:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:06:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:06:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:06:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:06:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:06:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.61 bytes/sec: 33 MiB/s 2026-02-22T20:06:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster1 mirror test_force_promote 2026-02-22T20:06:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:04.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster1 mirror test_force_promote 2026-02-22T20:06:04.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:04.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:04.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:04.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote 2026-02-22T20:06:04.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote 2026-02-22T20:06:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:06:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test_force_promote 2026-02-22T20:06:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test_force_promote 2026-02-22T20:06:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote 2026-02-22T20:06:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote 2026-02-22T20:06:04.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote 2026-02-22T20:06:05.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:06:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: cloned images' 2026-02-22T20:06:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:06:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:06:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:06:05 TEST: cloned images 2026-02-22T20:06:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:06:05 TEST: cloned images 2026-02-22T20:06:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog ' - default' 2026-02-22T20:06:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:06:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:06:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:06:05 - default 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:06:05 - default 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ parent_image=test_parent 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ parent_snap=snap 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror_parent test_parent snapshot 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror_parent test_parent 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:06:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror_parent/test_parent 2026-02-22T20:06:05.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror_parent 2026-02-22T20:06:05.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:05.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:06:05.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:05.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror_parent test_parent snapshot 2026-02-22T20:06:05.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror_parent/test_parent snapshot 2026-02-22T20:06:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:06:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror_parent/test_parent 2026-02-22T20:06:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_parent': 2026-02-22T20:06:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:06:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:06:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 152aa0cb15d7 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.152aa0cb15d7 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:06:05 2026 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:06:05 2026 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:06:05 2026 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:06:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 0fd3f5d3-cb0d-4d2f-a0b4-adc3f69be3f4 2026-02-22T20:06:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:06:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror_parent test_parent 100 2026-02-22T20:06:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:06:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:06:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:06:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:06:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror_parent/test_parent --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:06:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:06:06.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:06:06.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.61 bytes/sec: 33 MiB/s 2026-02-22T20:06:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T20:06:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap 2026-02-22T20:06:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap 2026-02-22T20:06:07.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:06:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ protect_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T20:06:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap 2026-02-22T20:06:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap protect mirror_parent/test_parent@snap 2026-02-22T20:06:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image=test_clone 2026-02-22T20:06:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone 2026-02-22T20:06:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_pool=mirror_parent 2026-02-22T20:06:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_image=test_parent 2026-02-22T20:06:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_snap=snap 2026-02-22T20:06:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_pool=mirror 2026-02-22T20:06:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_image=test_clone 2026-02-22T20:06:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 6 2026-02-22T20:06:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone --image-feature layering 2026-02-22T20:06:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_clone 100 2026-02-22T20:06:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:06:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:06:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:06:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:06:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_clone --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:06:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:06:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:06:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 5000.16 bytes/sec: 20 MiB/s 2026-02-22T20:06:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:06:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_clone snapshot 2026-02-22T20:06:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:07.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:07.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone snapshot 2026-02-22T20:06:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:06:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_clone 2026-02-22T20:06:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_clone': 2026-02-22T20:06:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:06:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:06:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 153c7a31d117 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.153c7a31d117 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:06:07 2026 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:06:07 2026 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:06:07 2026 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: parent: mirror_parent/test_parent@snap 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: overlap: 128 MiB 2026-02-22T20:06:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:06:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:06:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: beeecf71-f4f5-48b3-acfe-f3cd3d7a0b43 2026-02-22T20:06:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:06:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror_parent test_parent 2026-02-22T20:06:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror_parent test_parent started 2026-02-22T20:06:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:06:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror_parent test_parent started 2026-02-22T20:06:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:06:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:06:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:06:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror_parent/test_parent 2026-02-22T20:06:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:06:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:06:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:06:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:06:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:06:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:06:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/test_parent 2026-02-22T20:06:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:06:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:06:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:09.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:09.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:06:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/test_parent 2026-02-22T20:06:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:06:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:06:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:06:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:06:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror_parent test_parent 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:06:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T20:06:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T20:06:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 5 2026-02-22T20:06:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror_parent test_parent 2026-02-22T20:06:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:06:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror_parent-test_parent.status 2026-02-22T20:06:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror_parent-test_parent.status 2026-02-22T20:06:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_parent-test_parent.status 2026-02-22T20:06:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent test_parent /tmp/tmp.5OeMBFOFTF/cluster2-mirror_parent-test_parent.status 2026-02-22T20:06:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_parent-test_parent.status 2026-02-22T20:06:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/test_parent --format xml 2026-02-22T20:06:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:06:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=5 2026-02-22T20:06:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:06:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:06:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-22T20:06:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=3 2026-02-22T20:06:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 5 = 3 2026-02-22T20:06:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:06:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-22T20:06:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:10.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:06:10.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:06:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:06:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:06:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:06:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 5 = '' 2026-02-22T20:06:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.8 2026-02-22T20:06:11.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-22T20:06:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:06:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:06:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:06:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:06:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:06:11.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 5 = '' 2026-02-22T20:06:11.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:11.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1.6 2026-02-22T20:06:13.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:13.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:13.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:13.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.status 2026-02-22T20:06:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-22T20:06:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=5 2026-02-22T20:06:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 5 = 5 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror_parent test_parent 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:13.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:06:13.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:13.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:13.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:13.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:14.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying '' '' 2026-02-22T20:06:14.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:14.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:14.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:06:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror_parent/test_parent 2026-02-22T20:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_parent: 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 0fd3f5d3-cb0d-4d2f-a0b4-adc3f69be3f4 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790766,"replay_state":"idle"} 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:06:08 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:06:13' 2026-02-22T20:06:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:06:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:06:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:06:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790766,"replay_state":"idle"} 2026-02-22T20:06:14.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:06:14.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:06:14.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:06:14.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:06:14.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:06:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:06:14.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:06:14.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:06:14.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:06:08' 2026-02-22T20:06:14.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror_parent test_parent up+replaying '2026-02-22 20:06:08' 2026-02-22T20:06:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:06:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:06:08' 2026-02-22T20:06:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format xml 2026-02-22T20:06:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK143244349mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284352mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354359mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414358mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_parent0fd3f5d3-cb0d-4d2f-a0b4-adc3f69be3f4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790766,"replay_state":"idle"}43244349mirror.0trial0982026-02-22 20:06:08cluster2-DNE3555de88-7c67-446a-96b5-79f2a20ada79up+stoppedlocal image is primary2026-02-22 20:06:13' 2026-02-22T20:06:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:06:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/last_update' 2026-02-22T20:06:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:06:08' 2026-02-22T20:06:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/state' 2026-02-22T20:06:14.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:06:14.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:06:14.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:06:14.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:06:14.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:14.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror_parent test_parent 2026-02-22T20:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror_parent-test_parent.export 2026-02-22T20:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror_parent-test_parent.export 2026-02-22T20:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:14.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_parent-test_parent.export 2026-02-22T20:06:14.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror_parent-test_parent.export 2026-02-22T20:06:14.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror_parent-test_parent.export 2026-02-22T20:06:14.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.export 2026-02-22T20:06:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror_parent-test_parent.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.export 2026-02-22T20:06:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror_parent/test_parent /tmp/tmp.5OeMBFOFTF/cluster2-mirror_parent-test_parent.export 2026-02-22T20:06:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror_parent/test_parent /tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.export 2026-02-22T20:06:14.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror_parent-test_parent.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.export 2026-02-22T20:06:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror_parent-test_parent.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_parent-test_parent.export 2026-02-22T20:06:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone 2026-02-22T20:06:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone started 2026-02-22T20:06:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_clone started 2026-02-22T20:06:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:06:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:06:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:06:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone 2026-02-22T20:06:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:06:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:06:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone 2026-02-22T20:06:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone'] 2026-02-22T20:06:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone 2026-02-22T20:06:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:06:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:06:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:06:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:06:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:06:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:06:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:06:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone 2026-02-22T20:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:06:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:06:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:06:16.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:06:16.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:06:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:06:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:06:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_clone 2026-02-22T20:06:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_clone 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_clone 2026-02-22T20:06:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 150 2026-02-22T20:06:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_clone 2026-02-22T20:06:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:06:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_clone.status 2026-02-22T20:06:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_clone.status 2026-02-22T20:06:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_clone.status 2026-02-22T20:06:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_clone.status 2026-02-22T20:06:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_clone.status 2026-02-22T20:06:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.status 2026-02-22T20:06:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_clone /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_clone.status 2026-02-22T20:06:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_clone.status 2026-02-22T20:06:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_clone --format xml 2026-02-22T20:06:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:06:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=150 2026-02-22T20:06:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:06:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:06:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.status 2026-02-22T20:06:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.status 2026-02-22T20:06:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-02-22T20:06:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:16.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=149 2026-02-22T20:06:16.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 150 = 149 2026-02-22T20:06:16.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:06:16.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.status 2026-02-22T20:06:16.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:16.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.status 2026-02-22T20:06:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-02-22T20:06:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=150 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 150 = 150 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_clone 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_clone up+replaying 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:06:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_clone up+replaying '' '' 2026-02-22T20:06:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:06:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_clone 2026-02-22T20:06:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_clone: 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: beeecf71-f4f5-48b3-acfe-f3cd3d7a0b43 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790768,"replay_state":"idle"} 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:06:14 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:06:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:06:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:06:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:06:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:06:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790768,"replay_state":"idle"} 2026-02-22T20:06:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:06:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:06:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:06:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:06:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:06:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:06:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:06:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:06:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:06:14' 2026-02-22T20:06:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_clone up+replaying '2026-02-22 20:06:14' 2026-02-22T20:06:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:06:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:06:14' 2026-02-22T20:06:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:06:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:05:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:06:18test_clonebeeecf71-f4f5-48b3-acfe-f3cd3d7a0b43up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790768,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 20:06:14cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:06:18' 2026-02-22T20:06:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:06:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/last_update' 2026-02-22T20:06:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:06:14' 2026-02-22T20:06:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/state' 2026-02-22T20:06:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:06:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:06:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:06:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:06:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test_clone 2026-02-22T20:06:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:06:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_clone.export 2026-02-22T20:06:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_clone.export 2026-02-22T20:06:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_clone.export 2026-02-22T20:06:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_clone.export 2026-02-22T20:06:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_clone.export 2026-02-22T20:06:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.export 2026-02-22T20:06:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_clone.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.export 2026-02-22T20:06:18.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test_clone /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_clone.export 2026-02-22T20:06:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test_clone /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.export 2026-02-22T20:06:18.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:18.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_clone.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.export 2026-02-22T20:06:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_clone.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_clone.export 2026-02-22T20:06:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test_clone 2026-02-22T20:06:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test_clone 2026-02-22T20:06:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone 2026-02-22T20:06:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone 2026-02-22T20:06:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone 2026-02-22T20:06:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:06:19.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:19.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog ' - clone v1' 2026-02-22T20:06:19.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:06:19.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:06:19.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:06:19 - clone v1 2026-02-22T20:06:19.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:06:19 - clone v1 2026-02-22T20:06:19.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap mirror test_clone1 snapshot 2026-02-22T20:06:19.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_pool=mirror_parent 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_image=test_parent 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_snap=snap 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_pool=mirror 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_image=test_clone1 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 6 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image cluster1 mirror_parent test_parent snap mirror test_clone1 2026-02-22T20:06:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_pool=mirror_parent 2026-02-22T20:06:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_image=test_parent 2026-02-22T20:06:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_snap=snap 2026-02-22T20:06:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_pool=mirror 2026-02-22T20:06:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_image=test_clone1 2026-02-22T20:06:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 6 2026-02-22T20:06:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap mirror/test_clone1 --image-feature layering 2026-02-22T20:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster1 mirror test_clone1 snapshot 2026-02-22T20:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone1 2026-02-22T20:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/test_clone1 snapshot 2026-02-22T20:06:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:06:19.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 info mirror/test_clone1 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_clone1': 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 155824774fd3 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.155824774fd3 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:06:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:06:19 2026 2026-02-22T20:06:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:06:19 2026 2026-02-22T20:06:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:06:19 2026 2026-02-22T20:06:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: parent: mirror_parent/test_parent@snap 2026-02-22T20:06:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: overlap: 128 MiB 2026-02-22T20:06:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:06:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:06:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 7916969a-7173-4f5e-bed2-93d045a74baf 2026-02-22T20:06:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap mirror test_clone_v1 snapshot --rbd-default-clone-format 1 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_pool=mirror_parent 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_image=test_parent 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_snap=snap 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_pool=mirror 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_image=test_clone_v1 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 6 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:06:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone_v1 --rbd-default-clone-format 1 2026-02-22T20:06:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_pool=mirror_parent 2026-02-22T20:06:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_image=test_parent 2026-02-22T20:06:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_snap=snap 2026-02-22T20:06:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_pool=mirror 2026-02-22T20:06:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_image=test_clone_v1 2026-02-22T20:06:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 6 2026-02-22T20:06:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone_v1 --image-feature layering --rbd-default-clone-format 1 2026-02-22T20:06:19.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:06:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:06:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_clone_v1 snapshot 2026-02-22T20:06:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v1 2026-02-22T20:06:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone_v1 snapshot 2026-02-22T20:06:20.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:06:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_clone_v1 2026-02-22T20:06:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_clone_v1': 2026-02-22T20:06:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:06:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:06:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:06:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 156052b00088 2026-02-22T20:06:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.156052b00088 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:06:19 2026 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:06:19 2026 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:06:19 2026 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: parent: mirror_parent/test_parent@snap 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: overlap: 128 MiB 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:06:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: da0623d4-7e40-45ce-b4e6-f664254c9aae 2026-02-22T20:06:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:06:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_clone_format cluster2 mirror test_clone_v1 2026-02-22T20:06:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:06:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_clone_v1 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v1 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ awk 'BEGIN { 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: format = 1 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: $1 == "parent:" { 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: parent = $2 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: /op_features: .*clone-child/ { 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: format = 2 2026-02-22T20:06:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: END { 2026-02-22T20:06:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: if (!parent) exit 1 2026-02-22T20:06:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: print format 2026-02-22T20:06:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: }' 2026-02-22T20:06:20.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 1 = 1 2026-02-22T20:06:20.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v1 2026-02-22T20:06:20.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:20.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:20.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v1 2026-02-22T20:06:20.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v1 started 2026-02-22T20:06:20.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v1 2026-02-22T20:06:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:06:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_clone_v1 started 2026-02-22T20:06:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v1 2026-02-22T20:06:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:06:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:06:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:06:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:06:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:06:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:06:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:06:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:06:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:06:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:21.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:06:21.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T20:06:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:06:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:06:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:06:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:06:21.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:06:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:21.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T20:06:21.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:21.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:06:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:06:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:06:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:06:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:06:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:06:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T20:06:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:06:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:06:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:06:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:06:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:06:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:21.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T20:06:21.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T20:06:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:06:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:06:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:06:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T20:06:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:06:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:06:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:06:22.696 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T20:06:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:06:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:06:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:06:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:06:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:06:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:06:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v1 2026-02-22T20:06:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:06:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:06:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:06:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:06:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:06:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:06:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:06:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_clone_format cluster1 mirror test_clone_v1 2026-02-22T20:06:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:06:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_clone_v1 2026-02-22T20:06:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v1 2026-02-22T20:06:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ awk 'BEGIN { 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: format = 1 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: $1 == "parent:" { 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: parent = $2 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: /op_features: .*clone-child/ { 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: format = 2 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: END { 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: if (!parent) exit 1 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: print format 2026-02-22T20:06:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: }' 2026-02-22T20:06:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 1 = 1 2026-02-22T20:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test_clone_v1 2026-02-22T20:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v1 2026-02-22T20:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test_clone_v1 2026-02-22T20:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v1 2026-02-22T20:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v1 2026-02-22T20:06:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v1 2026-02-22T20:06:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:06:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster1 mirror test_clone1 2026-02-22T20:06:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone1 2026-02-22T20:06:23.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:23.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:23.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster1 mirror test_clone1 2026-02-22T20:06:23.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:23.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:23.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone1 2026-02-22T20:06:23.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone1 2026-02-22T20:06:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone1 2026-02-22T20:06:23.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:06:23.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ unprotect_snapshot_retry cluster2 mirror_parent test_parent snap 2026-02-22T20:06:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap 2026-02-22T20:06:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ unprotect_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T20:06:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap 2026-02-22T20:06:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap unprotect mirror_parent/test_parent@snap 2026-02-22T20:06:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: unprotecting snap failed: (16) Device or resource busy 2026-02-22T20:06:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ unprotect_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T20:06:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap 2026-02-22T20:06:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap unprotect mirror_parent/test_parent@snap 2026-02-22T20:06:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T20:06:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap 2026-02-22T20:06:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap 2026-02-22T20:06:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing snap: 100% complete...done. 2026-02-22T20:06:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog ' - clone v2' 2026-02-22T20:06:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:06:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:06:25.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:06:25 - clone v2 2026-02-22T20:06:25.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:06:25 - clone v2 2026-02-22T20:06:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ parent_snap=snap_v2 2026-02-22T20:06:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T20:06:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_v2 2026-02-22T20:06:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-02-22T20:06:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:06:26.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:06:26.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T20:06:26.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:26.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:26.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:26.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T20:06:28.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 7 2026-02-22T20:06:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 snapshot --rbd-default-clone-format 2 2026-02-22T20:06:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_pool=mirror_parent 2026-02-22T20:06:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_image=test_parent 2026-02-22T20:06:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_snap=snap_v2 2026-02-22T20:06:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_pool=mirror 2026-02-22T20:06:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_image=test_clone_v2 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 6 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_pool=mirror_parent 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_image=test_parent 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_snap=snap_v2 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_pool=mirror 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_image=test_clone_v2 2026-02-22T20:06:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 6 2026-02-22T20:06:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering --rbd-default-clone-format 2 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_clone_v2 snapshot 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone_v2 snapshot 2026-02-22T20:06:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:06:29.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_clone_v2 2026-02-22T20:06:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_clone_v2': 2026-02-22T20:06:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:06:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:06:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 15811cd6b4c9 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.15811cd6b4c9 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, operations 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: clone-child 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:06:28 2026 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:06:28 2026 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:06:28 2026 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: parent: mirror_parent/test_parent@snap_v2 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: overlap: 128 MiB 2026-02-22T20:06:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:06:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:06:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: bd8fd23a-e584-4cfb-9f81-175b58d83ae0 2026-02-22T20:06:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:06:29.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_clone_format cluster2 mirror test_clone_v2 2026-02-22T20:06:29.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:06:29.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:06:29.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_clone_v2 2026-02-22T20:06:29.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v2 2026-02-22T20:06:29.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ awk 'BEGIN { 2026-02-22T20:06:29.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: format = 1 2026-02-22T20:06:29.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:29.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: $1 == "parent:" { 2026-02-22T20:06:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: parent = $2 2026-02-22T20:06:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: /op_features: .*clone-child/ { 2026-02-22T20:06:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: format = 2 2026-02-22T20:06:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: END { 2026-02-22T20:06:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: if (!parent) exit 1 2026-02-22T20:06:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: print format 2026-02-22T20:06:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: }' 2026-02-22T20:06:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 2 = 2 2026-02-22T20:06:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v2 2026-02-22T20:06:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v2 started 2026-02-22T20:06:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:06:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:30.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_clone_v2 started 2026-02-22T20:06:30.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:30.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:30.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:06:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:06:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:06:30.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:30.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:06:30.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:06:30.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:30.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:06:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:06:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:06:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:06:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:30.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:30.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:30.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:30.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:06:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T20:06:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:30.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:06:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:06:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:06:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:06:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T20:06:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:30.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:30.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:06:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:06:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:06:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:06:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:06:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:06:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T20:06:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:06:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:06:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:06:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:06:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:06:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:06:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:06:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:30.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T20:06:30.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:30.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:30.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:30.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T20:06:31.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:06:31.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:06:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:06:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:31.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:31.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:31.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:31.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T20:06:31.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:31.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:06:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:06:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:06:31.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:31.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:31.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:31.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-22T20:06:31.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:06:31.745 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:06:31.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:06:31.746 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:06:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:06:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:06:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_clone_format cluster1 mirror test_clone_v2 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_clone_v2 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v2 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ awk 'BEGIN { 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: format = 1 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: $1 == "parent:" { 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: parent = $2 2026-02-22T20:06:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:31.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: /op_features: .*clone-child/ { 2026-02-22T20:06:31.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: format = 2 2026-02-22T20:06:31.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: } 2026-02-22T20:06:31.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: END { 2026-02-22T20:06:31.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: if (!parent) exit 1 2026-02-22T20:06:31.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: print format 2026-02-22T20:06:31.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: }' 2026-02-22T20:06:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 2 = 2 2026-02-22T20:06:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T20:06:31.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:31.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:31.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:31.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_v2 2026-02-22T20:06:31.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-02-22T20:06:31.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing snap: 100% complete...done. 2026-02-22T20:06:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:06:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T20:06:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T20:06:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 8 2026-02-22T20:06:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T20:06:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap_v2 2026-02-22T20:06:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-22T20:06:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T20:06:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 6 bb6ca8dd-5e16-4e9b-8444-c0cc4e496b46 128 MiB Sun Feb 22 20:06:26 2026 trash (user snap_v2) 2026-02-22T20:06:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:06:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap_v2 2026-02-22T20:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T20:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T20:06:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 6 82e72403-3d46-4512-85c5-d735b12ae5e2 128 MiB Sun Feb 22 20:06:27 2026 trash (user snap_v2) 2026-02-22T20:06:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test_clone_v2 2026-02-22T20:06:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test_clone_v2 2026-02-22T20:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v2 2026-02-22T20:06:34.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v2 2026-02-22T20:06:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:06:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test_clone_v2 deleted 2026-02-22T20:06:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:06:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:06:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:06:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-02-22T20:06:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:06:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_clone_v2 2026-02-22T20:06:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-22T20:06:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:06:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-02-22T20:06:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T20:06:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:06:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:06:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test_clone_v2 deleted '' 2026-02-22T20:06:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_clone_v2 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-22T20:06:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:06:36.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-02-22T20:06:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:06:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:06:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:06:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T20:06:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T20:06:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:36.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap_v2 2026-02-22T20:06:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-22T20:06:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T20:06:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:06:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap_v2 2026-02-22T20:06:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T20:06:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T20:06:37.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:37.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:37.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog ' - clone v2 non-primary' 2026-02-22T20:06:37.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:06:37.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:06:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:06:37 - clone v2 non-primary 2026-02-22T20:06:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:06:37 - clone v2 non-primary 2026-02-22T20:06:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T20:06:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_v2 2026-02-22T20:06:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-02-22T20:06:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:06:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:06:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T20:06:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T20:06:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 10 2026-02-22T20:06:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_present cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap_v2 2026-02-22T20:06:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:06:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror_parent/test_parent@snap_v2 2026-02-22T20:06:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_parent': 2026-02-22T20:06:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:06:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 3 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 10fd6794438d 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.10fd6794438d 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, operations, non-primary 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: snap-trash 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:06:08 2026 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:06:08 2026 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:06:08 2026 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: protected: False 2026-02-22T20:06:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:06:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:06:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 0fd3f5d3-cb0d-4d2f-a0b4-adc3f69be3f4 2026-02-22T20:06:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: false 2026-02-22T20:06:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 snapshot --rbd-default-clone-format 2 2026-02-22T20:06:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_pool=mirror_parent 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_image=test_parent 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_snap=snap_v2 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_pool=mirror 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_image=test_clone_v2 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 6 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ clone_image cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-02-22T20:06:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_pool=mirror_parent 2026-02-22T20:06:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_image=test_parent 2026-02-22T20:06:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local parent_snap=snap_v2 2026-02-22T20:06:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_pool=mirror 2026-02-22T20:06:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local clone_image=test_clone_v2 2026-02-22T20:06:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 6 2026-02-22T20:06:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering --rbd-default-clone-format 2 2026-02-22T20:06:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:06:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:06:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster1 mirror test_clone_v2 snapshot 2026-02-22T20:06:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/test_clone_v2 snapshot 2026-02-22T20:06:41.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:06:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-22T20:06:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_clone_v2': 2026-02-22T20:06:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:06:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:06:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:06:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 157c7efcc49d 2026-02-22T20:06:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.157c7efcc49d 2026-02-22T20:06:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:06:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, operations 2026-02-22T20:06:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: clone-child 2026-02-22T20:06:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:06:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:06:41 2026 2026-02-22T20:06:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:06:41 2026 2026-02-22T20:06:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:06:41 2026 2026-02-22T20:06:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: parent: mirror_parent/test_parent@snap_v2 2026-02-22T20:06:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: overlap: 128 MiB 2026-02-22T20:06:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:06:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:06:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: b00ebc41-0c0b-4f09-930e-46ee629f5732 2026-02-22T20:06:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:06:41.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T20:06:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_v2 2026-02-22T20:06:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-02-22T20:06:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing snap: 100% complete...done. 2026-02-22T20:06:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T20:06:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T20:06:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap_v2 2026-02-22T20:06:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-22T20:06:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T20:06:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:06:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T20:06:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T20:06:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 11 2026-02-22T20:06:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:06:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap_v2 2026-02-22T20:06:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T20:06:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T20:06:45.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 9 2974d11f-e8b0-46c4-8dc3-fb11948cb9d3 128 MiB Sun Feb 22 20:06:39 2026 trash (user snap_v2) 2026-02-22T20:06:45.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster1 mirror test_clone_v2 2026-02-22T20:06:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster1 mirror test_clone_v2 2026-02-22T20:06:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_clone_v2 2026-02-22T20:06:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone_v2 2026-02-22T20:06:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone_v2 2026-02-22T20:06:46.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:06:46.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:46.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:46.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:46.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:06:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T20:06:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap_v2 2026-02-22T20:06:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T20:06:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T20:06:47.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:47.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:47.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror_parent test_parent 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror_parent test_parent 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror_parent 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_parent 2026-02-22T20:06:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror_parent/test_parent 2026-02-22T20:06:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror_parent/test_parent 2026-02-22T20:06:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:06:48.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:48.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: data pool' 2026-02-22T20:06:48.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:06:48.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:06:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:06:48 TEST: data pool 2026-02-22T20:06:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:06:48 TEST: data pool 2026-02-22T20:06:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ dp_image=test_data_pool 2026-02-22T20:06:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test_data_pool snapshot 128 --data-pool mirror_parent 2026-02-22T20:06:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test_data_pool 128 --data-pool mirror_parent 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:06:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 128 ']' 2026-02-22T20:06:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=128 2026-02-22T20:06:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:06:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering --data-pool mirror_parent mirror/test_data_pool 2026-02-22T20:06:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:06:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:06:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_data_pool snapshot 2026-02-22T20:06:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:06:48.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_data_pool snapshot 2026-02-22T20:06:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:06:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_data_pool 2026-02-22T20:06:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_data_pool': 2026-02-22T20:06:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:06:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 15b43c00291a 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: data_pool: mirror_parent 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.3.15b43c00291a 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, data-pool 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:06:48 2026 2026-02-22T20:06:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:06:48 2026 2026-02-22T20:06:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:06:48 2026 2026-02-22T20:06:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:06:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:06:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 251551a4-84ef-434b-948c-969e94b82253 2026-02-22T20:06:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:06:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_data_pool cluster2 mirror test_data_pool 2026-02-22T20:06:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:06:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:06:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_data_pool 2026-02-22T20:06:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 info mirror/test_data_pool 2026-02-22T20:06:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T20:06:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ data_pool=mirror_parent 2026-02-22T20:06:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test mirror_parent = mirror_parent 2026-02-22T20:06:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_data_pool 2026-02-22T20:06:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_data_pool started 2026-02-22T20:06:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:06:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:06:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_data_pool started 2026-02-22T20:06:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:06:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:06:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_data_pool 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:06:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-02-22T20:06:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:06:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:50.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:50.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:06:50.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:50.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:50.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:50.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-02-22T20:06:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T20:06:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-02-22T20:06:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:06:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:06:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:06:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:06:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-02-22T20:06:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T20:06:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_data_pool 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:06:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:50.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:50.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:06:50.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:06:50.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:50.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:50.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:06:50.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:06:50.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:06:50.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:50.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:50.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:06:50.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_data_pool 2026-02-22T20:06:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T20:06:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_data_pool 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:06:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:06:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:06:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:06:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:06:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:06:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:06:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:06:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:06:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_data_pool 2026-02-22T20:06:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T20:06:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T20:06:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:06:51.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:06:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-02-22T20:06:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:06:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:51.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:06:51.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:06:51.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:06:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:06:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-02-22T20:06:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T20:06:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:06:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:06:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:06:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-02-22T20:06:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:06:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:06:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:06:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:06:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:06:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:06:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:06:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:06:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:51.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:06:51.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-02-22T20:06:51.707 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:06:51.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:06:51.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:06:51.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:06:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:06:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:06:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:06:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_data_pool cluster1 mirror test_data_pool 2026-02-22T20:06:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:06:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:06:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_data_pool 2026-02-22T20:06:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 info mirror/test_data_pool 2026-02-22T20:06:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T20:06:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ data_pool=mirror_parent 2026-02-22T20:06:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test mirror_parent = mirror_parent 2026-02-22T20:06:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test_data_pool snap1 2026-02-22T20:06:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:51.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:06:51.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap1 2026-02-22T20:06:52.241 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:06:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-02-22T20:06:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:06:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:06:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:06:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:06:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:06:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:06:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:06:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.59 bytes/sec: 33 MiB/s 2026-02-22T20:06:52.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test_data_pool snap2 2026-02-22T20:06:52.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:52.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:52.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:52.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:06:52.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap2 2026-02-22T20:06:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:06:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-02-22T20:06:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:06:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:06:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:06:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:06:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:06:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:06:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:06:53.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.59 bytes/sec: 33 MiB/s 2026-02-22T20:06:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_data_pool 2026-02-22T20:06:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:06:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:06:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:06:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_data_pool 2026-02-22T20:06:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_data_pool 2026-02-22T20:06:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 15 2026-02-22T20:06:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_data_pool 2026-02-22T20:06:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:06:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T20:06:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_data_pool.status 2026-02-22T20:06:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool.status 2026-02-22T20:06:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_data_pool.status 2026-02-22T20:06:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_data_pool.status 2026-02-22T20:06:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.status 2026-02-22T20:06:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_data_pool /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool.status 2026-02-22T20:06:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:54.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:54.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool.status 2026-02-22T20:06:54.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_data_pool --format xml 2026-02-22T20:06:54.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=15 2026-02-22T20:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:06:54.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.status 2026-02-22T20:06:54.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:54.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.status 2026-02-22T20:06:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-22T20:06:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:54.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=12 2026-02-22T20:06:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 15 = 12 2026-02-22T20:06:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:06:54.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.status 2026-02-22T20:06:54.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:54.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:54.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:54.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.status 2026-02-22T20:06:54.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-22T20:06:54.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:06:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:54.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:06:54.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:06:54.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:06:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:06:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 15 = '' 2026-02-22T20:06:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.8 2026-02-22T20:06:55.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.status 2026-02-22T20:06:55.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:55.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:55.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:55.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.status 2026-02-22T20:06:55.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-22T20:06:55.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:55.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:06:55.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:06:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:06:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:06:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:06:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 15 = '' 2026-02-22T20:06:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:06:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1.6 2026-02-22T20:06:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.status 2026-02-22T20:06:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.status 2026-02-22T20:06:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-22T20:06:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:06:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:06:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=15 2026-02-22T20:06:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 15 = 15 2026-02-22T20:06:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_data_pool 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_data_pool up+replaying 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:06:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:06:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_data_pool up+replaying '' '' 2026-02-22T20:06:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:06:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:06:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:06:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_data_pool 2026-02-22T20:06:58.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_data_pool: 2026-02-22T20:06:58.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 251551a4-84ef-434b-948c-969e94b82253 2026-02-22T20:06:58.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:06:58.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790809,"remote_snapshot_timestamp":1771790809,"replay_state":"idle"} 2026-02-22T20:06:58.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:06:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:06:54 2026-02-22T20:06:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:06:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:06:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:06:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:06:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:06:58' 2026-02-22T20:06:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:06:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:06:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:06:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790809,"remote_snapshot_timestamp":1771790809,"replay_state":"idle"} 2026-02-22T20:06:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:06:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:06:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:06:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:06:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:06:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:06:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:06:58.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:06:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:06:54' 2026-02-22T20:06:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_data_pool up+replaying '2026-02-22 20:06:54' 2026-02-22T20:06:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:06:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:06:58.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:06:54' 2026-02-22T20:06:58.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:06:58.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:06:58.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:06:58.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:06:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:06:58test_data_pool251551a4-84ef-434b-948c-969e94b82253up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790809,"remote_snapshot_timestamp":1771790809,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 20:06:54cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:06:58' 2026-02-22T20:06:58.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:06:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-02-22T20:06:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:06:54' 2026-02-22T20:06:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-02-22T20:06:58.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:06:58.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:06:58.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:06:58.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:06:58.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test_data_pool@snap1 2026-02-22T20:06:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool@snap1 2026-02-22T20:06:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:06:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:58.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:58.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap1 /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:58.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap1 /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T20:06:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test_data_pool@snap2 2026-02-22T20:06:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool@snap2 2026-02-22T20:06:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:06:58.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_data_pool@snap2.export 2026-02-22T20:06:58.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_data_pool@snap2.export 2026-02-22T20:06:58.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:58.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap2.export 2026-02-22T20:06:58.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_data_pool@snap2.export 2026-02-22T20:06:58.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_data_pool@snap2.export 2026-02-22T20:06:58.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T20:06:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T20:06:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap2 /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap2.export 2026-02-22T20:06:58.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap2 /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T20:06:59.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T20:06:59.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T20:06:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test_data_pool 2026-02-22T20:06:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:06:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_data_pool.export 2026-02-22T20:06:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_data_pool.export 2026-02-22T20:06:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool.export 2026-02-22T20:06:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_data_pool.export 2026-02-22T20:06:59.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_data_pool.export 2026-02-22T20:06:59.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:06:59.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.export 2026-02-22T20:06:59.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.export 2026-02-22T20:06:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool.export 2026-02-22T20:06:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.export 2026-02-22T20:06:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:06:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.export 2026-02-22T20:06:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_data_pool.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_data_pool.export 2026-02-22T20:06:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:06:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test_data_pool 2026-02-22T20:06:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:06:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:06:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test_data_pool 2026-02-22T20:06:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:06:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:06:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:06:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_data_pool 2026-02-22T20:07:01.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T20:07:01.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test_data_pool 2026-02-22T20:07:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:07:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: disable mirroring / delete non-primary image' 2026-02-22T20:07:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:07:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:07:02 TEST: disable mirroring / delete non-primary image 2026-02-22T20:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:07:02 TEST: disable mirroring / delete non-primary image 2026-02-22T20:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image2=test2 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image3=test3 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image4=test4 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image5=test5 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test2 snapshot 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test2 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:07:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test2 2026-02-22T20:07:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:07:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:07:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test2 snapshot 2026-02-22T20:07:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test2 snapshot 2026-02-22T20:07:03.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:07:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test2 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test2': 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 15e1b99f91cb 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.15e1b99f91cb 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:02 2026 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:02 2026 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:02 2026 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: ee85192d-2615-462e-b5b7-7cd9b44480af 2026-02-22T20:07:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:07:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T20:07:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:03.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:03.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:03.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.59 bytes/sec: 33 MiB/s 2026-02-22T20:07:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test2 snap1 2026-02-22T20:07:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap1 2026-02-22T20:07:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:07:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test2 snap2 2026-02-22T20:07:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap2 2026-02-22T20:07:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:07:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' test2 = test4 ']' 2026-02-22T20:07:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' test2 = test5 ']' 2026-02-22T20:07:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T20:07:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.59 bytes/sec: 33 MiB/s 2026-02-22T20:07:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:07:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test2 2026-02-22T20:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test2 2026-02-22T20:07:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 157 2026-02-22T20:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test2 present 2026-02-22T20:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=present 2026-02-22T20:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T20:07:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test2 2026-02-22T20:07:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T20:07:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id=10f34dbaf082 2026-02-22T20:07:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:07:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:07:06.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test2 present 10f34dbaf082 2026-02-22T20:07:06.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:06.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:06.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:06.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:07:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=10f34dbaf082 2026-02-22T20:07:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:07:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:07:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T20:07:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test2 2026-02-22T20:07:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T20:07:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=10f34dbaf082 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 10f34dbaf082 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 10f34dbaf082 -o 10f34dbaf082 = 10f34dbaf082 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = present 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_present cluster1 mirror test2 snap2 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap2 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:07:06.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-22T20:07:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T20:07:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ continue 2026-02-22T20:07:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:07:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:07:09.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-22T20:07:09.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test2': 2026-02-22T20:07:09.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:09.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:09.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 4 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 10f34dbaf082 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.10f34dbaf082 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, operations, non-primary 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: snap-trash 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:04 2026 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:04 2026 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:04 2026 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: protected: False 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:09.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: ee85192d-2615-462e-b5b7-7cd9b44480af 2026-02-22T20:07:09.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: false 2026-02-22T20:07:09.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:09.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T20:07:09.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test3 snapshot 2026-02-22T20:07:09.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:09.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:09.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:09.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:09.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test3 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:07:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:07:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test3 2026-02-22T20:07:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:07:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:07:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test3 snapshot 2026-02-22T20:07:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test3 snapshot 2026-02-22T20:07:10.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:07:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test3 2026-02-22T20:07:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test3': 2026-02-22T20:07:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 15f9a05aeb9a 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.15f9a05aeb9a 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:09 2026 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:09 2026 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:09 2026 2026-02-22T20:07:10.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 6c5b6bb4-8ded-4597-8ded-7302a29545f2 2026-02-22T20:07:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:07:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test3 100 2026-02-22T20:07:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:10.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:10.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:10.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.59 bytes/sec: 33 MiB/s 2026-02-22T20:07:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test3 snap1 2026-02-22T20:07:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap1 2026-02-22T20:07:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:07:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test3 snap2 2026-02-22T20:07:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap2 2026-02-22T20:07:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:07:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' test3 = test4 ']' 2026-02-22T20:07:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' test3 = test5 ']' 2026-02-22T20:07:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test3 100 2026-02-22T20:07:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:12.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 6250.19 bytes/sec: 24 MiB/s 2026-02-22T20:07:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:07:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test3 2026-02-22T20:07:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test3 2026-02-22T20:07:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 161 2026-02-22T20:07:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test3 present 2026-02-22T20:07:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=present 2026-02-22T20:07:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T20:07:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test3 2026-02-22T20:07:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T20:07:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id=10f8daa01dbf 2026-02-22T20:07:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:07:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:07:13.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test3 present 10f8daa01dbf 2026-02-22T20:07:13.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:13.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=10f8daa01dbf 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test3 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T20:07:13.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:13.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=10f8daa01dbf 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 10f8daa01dbf 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 10f8daa01dbf -o 10f8daa01dbf = 10f8daa01dbf 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = present 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_present cluster1 mirror test3 snap2 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap2 2026-02-22T20:07:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:07:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test3@snap2 2026-02-22T20:07:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T20:07:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ continue 2026-02-22T20:07:14.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:07:14.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:07:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test3@snap2 2026-02-22T20:07:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test3': 2026-02-22T20:07:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 4 2026-02-22T20:07:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 10f8daa01dbf 2026-02-22T20:07:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.10f8daa01dbf 2026-02-22T20:07:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:16.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, operations, non-primary 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: snap-trash 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:12 2026 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:12 2026 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:12 2026 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: protected: False 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 6c5b6bb4-8ded-4597-8ded-7302a29545f2 2026-02-22T20:07:16.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: false 2026-02-22T20:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T20:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test4 snapshot 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test4 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test4 2026-02-22T20:07:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:07:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:07:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test4 snapshot 2026-02-22T20:07:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test4 snapshot 2026-02-22T20:07:16.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:07:16.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test4 2026-02-22T20:07:16.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test4': 2026-02-22T20:07:16.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:16.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 1610eca9d803 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.1610eca9d803 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:16 2026 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:16 2026 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:16 2026 2026-02-22T20:07:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:16.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:16.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 5b08a743-df30-4940-acad-244a6eacc268 2026-02-22T20:07:16.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:07:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test4 100 2026-02-22T20:07:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.58 bytes/sec: 33 MiB/s 2026-02-22T20:07:16.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test4 snap1 2026-02-22T20:07:16.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:16.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:16.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap1 2026-02-22T20:07:17.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:07:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test4 snap2 2026-02-22T20:07:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap2 2026-02-22T20:07:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:07:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' test4 = test4 ']' 2026-02-22T20:07:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ protect_snapshot cluster2 mirror test4 snap1 2026-02-22T20:07:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:18.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:18.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:18.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap1 2026-02-22T20:07:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ protect_snapshot cluster2 mirror test4 snap2 2026-02-22T20:07:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap2 2026-02-22T20:07:18.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test4 100 2026-02-22T20:07:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:18.707 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:18.707 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:18.707 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:18.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:18.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:18.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.58 bytes/sec: 33 MiB/s 2026-02-22T20:07:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:07:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test4 2026-02-22T20:07:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test4 2026-02-22T20:07:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 165 2026-02-22T20:07:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test4 present 2026-02-22T20:07:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=present 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test4 2026-02-22T20:07:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T20:07:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:19.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id=10faecc06a0a 2026-02-22T20:07:19.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:07:19.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:07:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test4 present 10faecc06a0a 2026-02-22T20:07:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:07:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=10faecc06a0a 2026-02-22T20:07:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:07:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:07:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T20:07:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test4 2026-02-22T20:07:19.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T20:07:19.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=10faecc06a0a 2026-02-22T20:07:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 10faecc06a0a 2026-02-22T20:07:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 10faecc06a0a -o 10faecc06a0a = 10faecc06a0a 2026-02-22T20:07:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = present 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_present cluster1 mirror test4 snap2 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap2 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:07:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test4@snap2 2026-02-22T20:07:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test4': 2026-02-22T20:07:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 3 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 10faecc06a0a 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.10faecc06a0a 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, non-primary 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:18 2026 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:18 2026 2026-02-22T20:07:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:18 2026 2026-02-22T20:07:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: protected: True 2026-02-22T20:07:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 5b08a743-df30-4940-acad-244a6eacc268 2026-02-22T20:07:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: false 2026-02-22T20:07:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T20:07:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test5 snapshot 2026-02-22T20:07:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test5 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:07:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:07:20.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test5 2026-02-22T20:07:20.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:07:20.885 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:07:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test5 snapshot 2026-02-22T20:07:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test5 snapshot 2026-02-22T20:07:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:07:21.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test5 2026-02-22T20:07:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test5': 2026-02-22T20:07:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 162ee921fc64 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.162ee921fc64 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:20 2026 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:20 2026 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:20 2026 2026-02-22T20:07:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 9ab31a1a-c77b-4acd-a2d4-f39a01689e94 2026-02-22T20:07:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:07:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test5 100 2026-02-22T20:07:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:21.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:21.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.58 bytes/sec: 33 MiB/s 2026-02-22T20:07:21.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test5 snap1 2026-02-22T20:07:21.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:21.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:21.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:21.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:21.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap1 2026-02-22T20:07:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:07:22.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test5 snap2 2026-02-22T20:07:22.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:22.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:22.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:22.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:22.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap2 2026-02-22T20:07:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:07:23.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' test5 = test4 ']' 2026-02-22T20:07:23.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' test5 = test5 ']' 2026-02-22T20:07:23.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ protect_snapshot cluster2 mirror test5 snap1 2026-02-22T20:07:23.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:23.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:23.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:23.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:23.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap1 2026-02-22T20:07:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ protect_snapshot cluster2 mirror test5 snap2 2026-02-22T20:07:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap2 2026-02-22T20:07:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test5 100 2026-02-22T20:07:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:23.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 6250.19 bytes/sec: 24 MiB/s 2026-02-22T20:07:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:07:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test5 2026-02-22T20:07:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test5 2026-02-22T20:07:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 169 2026-02-22T20:07:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test5 present 2026-02-22T20:07:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=present 2026-02-22T20:07:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T20:07:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test5 2026-02-22T20:07:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T20:07:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id=10ed6e6a3604 2026-02-22T20:07:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:07:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:07:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test5 present 10ed6e6a3604 2026-02-22T20:07:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:07:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=10ed6e6a3604 2026-02-22T20:07:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:07:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:07:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T20:07:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test5 2026-02-22T20:07:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T20:07:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=10ed6e6a3604 2026-02-22T20:07:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 10ed6e6a3604 2026-02-22T20:07:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 10ed6e6a3604 -o 10ed6e6a3604 = 10ed6e6a3604 2026-02-22T20:07:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:07:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = present 2026-02-22T20:07:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_present cluster1 mirror test5 snap2 2026-02-22T20:07:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap2 2026-02-22T20:07:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:07:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test5@snap2 2026-02-22T20:07:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T20:07:25.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ continue 2026-02-22T20:07:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:07:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:07:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test5@snap2 2026-02-22T20:07:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test5': 2026-02-22T20:07:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 4 2026-02-22T20:07:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 10ed6e6a3604 2026-02-22T20:07:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.10ed6e6a3604 2026-02-22T20:07:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, operations, non-primary 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: snap-trash 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:23 2026 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:23 2026 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:23 2026 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: protected: True 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 9ab31a1a-c77b-4acd-a2d4-f39a01689e94 2026-02-22T20:07:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: false 2026-02-22T20:07:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-22T20:07:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=image 2026-02-22T20:07:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-22T20:07:27.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd: mirroring is already configured for image mode 2026-02-22T20:07:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image4} 2026-02-22T20:07:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ disable_mirror cluster2 mirror test2 2026-02-22T20:07:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:27.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test2 2026-02-22T20:07:28.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring disabled 2026-02-22T20:07:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image4} 2026-02-22T20:07:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ disable_mirror cluster2 mirror test4 2026-02-22T20:07:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test4 2026-02-22T20:07:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring disabled 2026-02-22T20:07:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ unprotect_snapshot cluster2 mirror test5 snap1 2026-02-22T20:07:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap1 2026-02-22T20:07:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ unprotect_snapshot cluster2 mirror test5 snap2 2026-02-22T20:07:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap2 2026-02-22T20:07:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image3} ${image5} 2026-02-22T20:07:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_snapshot cluster2 mirror test3 snap1 2026-02-22T20:07:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap1 2026-02-22T20:07:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing snap: 100% complete...done. 2026-02-22T20:07:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_snapshot cluster2 mirror test3 snap2 2026-02-22T20:07:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap2 2026-02-22T20:07:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing snap: 100% complete...done. 2026-02-22T20:07:31.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test3 2026-02-22T20:07:31.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:31.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:31.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:31.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:31.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:07:31.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test3 2026-02-22T20:07:31.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:31.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:31.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:31.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test3 2026-02-22T20:07:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test3 2026-02-22T20:07:32.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:07:32.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:32.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image3} ${image5} 2026-02-22T20:07:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_snapshot cluster2 mirror test5 snap1 2026-02-22T20:07:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:32.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap1 2026-02-22T20:07:33.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing snap: 100% complete...done. 2026-02-22T20:07:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_snapshot cluster2 mirror test5 snap2 2026-02-22T20:07:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap2 2026-02-22T20:07:34.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing snap: 100% complete...done. 2026-02-22T20:07:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test5 2026-02-22T20:07:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:07:34.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test5 2026-02-22T20:07:34.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:34.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:34.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:34.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test5 2026-02-22T20:07:34.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test5 2026-02-22T20:07:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:07:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T20:07:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test2 deleted 2026-02-22T20:07:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test2 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T20:07:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:35.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-22T20:07:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T20:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:07:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test2 deleted '' 2026-02-22T20:07:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:07:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:07:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:07:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T20:07:35.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:35.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:35.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test2 2026-02-22T20:07:35.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T20:07:35.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:35.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-22T20:07:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:07:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:07:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T20:07:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test3 deleted 2026-02-22T20:07:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test3 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T20:07:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-22T20:07:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T20:07:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:07:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:07:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test3 deleted '' 2026-02-22T20:07:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test3 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test3 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T20:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:35.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-22T20:07:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test4 deleted 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:07:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T20:07:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test4 2026-02-22T20:07:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T20:07:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T20:07:35.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T20:07:35.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:07:35.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test4 deleted '' 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test4 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T20:07:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:35.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T20:07:35.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:07:35.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:35.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:07:35.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:35.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T20:07:35.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test5 deleted 2026-02-22T20:07:35.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:35.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test5 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T20:07:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:35.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-22T20:07:35.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T20:07:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:07:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test5 deleted '' 2026-02-22T20:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test5 2026-02-22T20:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T20:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test5 2026-02-22T20:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T20:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:07:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-22T20:07:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:07:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:07:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:07:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: snapshot rename' 2026-02-22T20:07:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:07:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:07:35.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:07:35 TEST: snapshot rename 2026-02-22T20:07:35.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:07:35 TEST: snapshot rename 2026-02-22T20:07:35.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ snap_name=snap_rename 2026-02-22T20:07:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:07:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test2 2026-02-22T20:07:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test2 snapshot 2026-02-22T20:07:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:07:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test2 2026-02-22T20:07:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test2': 2026-02-22T20:07:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 3 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 15e1b99f91cb 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.15e1b99f91cb 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:02 2026 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:02 2026 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:02 2026 2026-02-22T20:07:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:36.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:36.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 1af103fa-aeff-4364-9b30-a5b3fc41f0f6 2026-02-22T20:07:36.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:07:36.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test2 snap_rename_0 2026-02-22T20:07:36.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:36.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:36.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:36.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_0 2026-02-22T20:07:36.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap_rename_0 2026-02-22T20:07:37.240 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:07:37.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 1 20 2026-02-22T20:07:37.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 1 - 1 2026-02-22T20:07:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_0 snap_rename_1 2026-02-22T20:07:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_0 2026-02-22T20:07:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_1 2026-02-22T20:07:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_0 mirror/test2@snap_rename_1 2026-02-22T20:07:37.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 2 - 1 2026-02-22T20:07:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_1 snap_rename_2 2026-02-22T20:07:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_1 2026-02-22T20:07:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_2 2026-02-22T20:07:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_1 mirror/test2@snap_rename_2 2026-02-22T20:07:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 3 - 1 2026-02-22T20:07:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_2 snap_rename_3 2026-02-22T20:07:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_2 2026-02-22T20:07:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_3 2026-02-22T20:07:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_2 mirror/test2@snap_rename_3 2026-02-22T20:07:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 4 - 1 2026-02-22T20:07:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_3 snap_rename_4 2026-02-22T20:07:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_3 2026-02-22T20:07:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_4 2026-02-22T20:07:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_3 mirror/test2@snap_rename_4 2026-02-22T20:07:37.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 5 - 1 2026-02-22T20:07:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_4 snap_rename_5 2026-02-22T20:07:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_4 2026-02-22T20:07:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_5 2026-02-22T20:07:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_4 mirror/test2@snap_rename_5 2026-02-22T20:07:37.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 6 - 1 2026-02-22T20:07:37.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_5 snap_rename_6 2026-02-22T20:07:37.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_5 2026-02-22T20:07:37.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_6 2026-02-22T20:07:37.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_5 mirror/test2@snap_rename_6 2026-02-22T20:07:37.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 7 - 1 2026-02-22T20:07:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_6 snap_rename_7 2026-02-22T20:07:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_6 2026-02-22T20:07:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_7 2026-02-22T20:07:37.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_6 mirror/test2@snap_rename_7 2026-02-22T20:07:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 8 - 1 2026-02-22T20:07:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_7 snap_rename_8 2026-02-22T20:07:37.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_7 2026-02-22T20:07:37.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_8 2026-02-22T20:07:37.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_7 mirror/test2@snap_rename_8 2026-02-22T20:07:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 9 - 1 2026-02-22T20:07:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_8 snap_rename_9 2026-02-22T20:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_8 2026-02-22T20:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_9 2026-02-22T20:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_8 mirror/test2@snap_rename_9 2026-02-22T20:07:37.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 10 - 1 2026-02-22T20:07:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_9 snap_rename_10 2026-02-22T20:07:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_9 2026-02-22T20:07:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_10 2026-02-22T20:07:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_9 mirror/test2@snap_rename_10 2026-02-22T20:07:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 11 - 1 2026-02-22T20:07:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_10 snap_rename_11 2026-02-22T20:07:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_10 2026-02-22T20:07:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_11 2026-02-22T20:07:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_10 mirror/test2@snap_rename_11 2026-02-22T20:07:37.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 12 - 1 2026-02-22T20:07:37.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_11 snap_rename_12 2026-02-22T20:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_11 2026-02-22T20:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_12 2026-02-22T20:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_11 mirror/test2@snap_rename_12 2026-02-22T20:07:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 13 - 1 2026-02-22T20:07:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_12 snap_rename_13 2026-02-22T20:07:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_12 2026-02-22T20:07:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_13 2026-02-22T20:07:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_12 mirror/test2@snap_rename_13 2026-02-22T20:07:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 14 - 1 2026-02-22T20:07:37.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_13 snap_rename_14 2026-02-22T20:07:37.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_13 2026-02-22T20:07:37.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_14 2026-02-22T20:07:37.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_13 mirror/test2@snap_rename_14 2026-02-22T20:07:37.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 15 - 1 2026-02-22T20:07:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_14 snap_rename_15 2026-02-22T20:07:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_14 2026-02-22T20:07:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_15 2026-02-22T20:07:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_14 mirror/test2@snap_rename_15 2026-02-22T20:07:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 16 - 1 2026-02-22T20:07:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_15 snap_rename_16 2026-02-22T20:07:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_15 2026-02-22T20:07:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_16 2026-02-22T20:07:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_15 mirror/test2@snap_rename_16 2026-02-22T20:07:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 17 - 1 2026-02-22T20:07:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_16 snap_rename_17 2026-02-22T20:07:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_16 2026-02-22T20:07:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_17 2026-02-22T20:07:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_16 mirror/test2@snap_rename_17 2026-02-22T20:07:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 18 - 1 2026-02-22T20:07:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_17 snap_rename_18 2026-02-22T20:07:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_17 2026-02-22T20:07:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_18 2026-02-22T20:07:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_17 mirror/test2@snap_rename_18 2026-02-22T20:07:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 19 - 1 2026-02-22T20:07:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_18 snap_rename_19 2026-02-22T20:07:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_18 2026-02-22T20:07:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_19 2026-02-22T20:07:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_18 mirror/test2@snap_rename_19 2026-02-22T20:07:37.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in `seq 1 20` 2026-02-22T20:07:37.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ expr 20 - 1 2026-02-22T20:07:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_19 snap_rename_20 2026-02-22T20:07:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap_rename_19 2026-02-22T20:07:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local new_snap=snap_rename_20 2026-02-22T20:07:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_19 mirror/test2@snap_rename_20 2026-02-22T20:07:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:07:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test2 2026-02-22T20:07:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test2 2026-02-22T20:07:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 172 2026-02-22T20:07:38.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_present cluster1 mirror test2 snap_rename_20 2026-02-22T20:07:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap_rename_20 2026-02-22T20:07:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:07:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:39.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-22T20:07:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T20:07:39.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ continue 2026-02-22T20:07:39.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:07:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:07:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-22T20:07:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test2': 2026-02-22T20:07:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 4 2026-02-22T20:07:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 10f3b5c5f1af 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.10f3b5c5f1af 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, non-primary 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:37 2026 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:37 2026 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:37 2026 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: protected: False 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:41.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 1af103fa-aeff-4364-9b30-a5b3fc41f0f6 2026-02-22T20:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: false 2026-02-22T20:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ unprotect_snapshot cluster2 mirror test4 snap1 2026-02-22T20:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:41.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:07:41.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap1 2026-02-22T20:07:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ unprotect_snapshot cluster2 mirror test4 snap2 2026-02-22T20:07:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap2 2026-02-22T20:07:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap2 2026-02-22T20:07:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image4} 2026-02-22T20:07:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test2 2026-02-22T20:07:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:41.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:41.426 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:07:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test2 2026-02-22T20:07:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test2 2026-02-22T20:07:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test2 2026-02-22T20:07:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing all snapshots: 33% complete... Removing all snapshots: 66% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T20:07:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test2 2026-02-22T20:07:45.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:07:45.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:45.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for i in ${image2} ${image4} 2026-02-22T20:07:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test4 2026-02-22T20:07:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:45.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:07:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test4 2026-02-22T20:07:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test4 2026-02-22T20:07:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test4 2026-02-22T20:07:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T20:07:47.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test4 2026-02-22T20:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:07:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T20:07:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: demote image while daemon is offline' 2026-02-22T20:07:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:07:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:07:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:07:47 TEST: demote image while daemon is offline 2026-02-22T20:07:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:07:47 TEST: demote image while daemon is offline 2026-02-22T20:07:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image=test_demote_image 2026-02-22T20:07:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test_demote_image snapshot 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test_demote_image 2026-02-22T20:07:47.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:47.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:47.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:47.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:47.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:47.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:07:47.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:07:47.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:07:47.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test_demote_image 2026-02-22T20:07:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:07:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:07:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_demote_image snapshot 2026-02-22T20:07:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:07:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_demote_image snapshot 2026-02-22T20:07:48.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:07:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_demote_image 2026-02-22T20:07:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_demote_image': 2026-02-22T20:07:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:07:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:07:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:07:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 16cd97bc6a85 2026-02-22T20:07:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.16cd97bc6a85 2026-02-22T20:07:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:07:47 2026 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:07:47 2026 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:07:47 2026 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: bfcca720-0a88-4106-99f7-e762d4fbade4 2026-02-22T20:07:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:07:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-22T20:07:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:48.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:48.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.57 bytes/sec: 33 MiB/s 2026-02-22T20:07:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_demote_image 2026-02-22T20:07:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T20:07:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:07:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:07:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:49.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T20:07:49.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:49.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:49.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:49.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:07:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:07:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:07:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:07:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:07:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:07:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:07:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:07:49.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:07:49.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:07:49.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:07:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:07:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:07:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:07:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:07:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:07:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:07:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:07:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:07:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:07:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:07:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:07:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:07:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:07:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:07:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:07:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:07:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:07:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:07:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:07:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:07:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:07:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:07:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:07:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:07:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:07:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:07:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:07:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:07:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:07:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:07:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:07:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:07:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:2 2026-02-22T20:07:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:07:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:2 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:2 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T20:07:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:07:49.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:07:49.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:07:49.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:07:49.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:07:49.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:07:49.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:07:49.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:07:49.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:07:49.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:07:49.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:07:49.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:07:49.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:49.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:49.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:3 2026-02-22T20:07:49.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:49.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:49.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T20:07:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:07:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:07:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:07:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:07:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:07:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:07:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:07:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T20:07:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:3 2026-02-22T20:07:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:07:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T20:07:49.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:3 2026-02-22T20:07:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:07:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:07:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T20:07:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:07:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:07:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:07:49.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:07:49.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T20:07:49.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:07:49.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:07:49.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:07:49.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:07:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:07:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:07:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:07:49.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:07:49.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T20:07:50.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:07:50.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:07:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:07:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:07:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:07:50.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:07:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:07:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:07:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:07:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:07:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:07:50.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:07:50.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:07:50.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:07:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:07:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:07:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:07:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:07:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:07:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:07:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:07:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:07:50.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:07:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:07:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:07:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:07:50.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:07:50.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:07:50.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:07:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:07:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:07:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_demote_image 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:07:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:07:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:07:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T20:07:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:07:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:07:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:07:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:07:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:07:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:07:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:07:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:07:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:07:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:07:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:07:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:07:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:07:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:07:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:07:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:07:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:07:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:07:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:07:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:07:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:07:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:07:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:07:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:07:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:07:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_demote_image 2026-02-22T20:07:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:07:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:07:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:07:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:07:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:07:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:07:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:07:51.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:07:51.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:07:51.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:07:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:07:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:07:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:07:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:07:51.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:07:51.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:07:51.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:07:51.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:51.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T20:07:51.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:07:51.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:51.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:51.907 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:51.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:07:51.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:07:51.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_demote_image 2026-02-22T20:07:51.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:51.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:51.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:51.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_demote_image 2026-02-22T20:07:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 174 2026-02-22T20:07:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T20:07:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:07:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T20:07:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-22T20:07:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:07:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:07:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-22T20:07:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-22T20:07:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:07:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:07:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:07:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:07:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:07:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:07:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:07:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=174 2026-02-22T20:07:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:07:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:07:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:07:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:07:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:07:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:07:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:07:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:07:52.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=173 2026-02-22T20:07:52.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 174 = 173 2026-02-22T20:07:52.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:07:52.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:07:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:07:52.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:52.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:52.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:52.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:07:52.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:07:52.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:07:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:07:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=174 2026-02-22T20:07:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 174 = 174 2026-02-22T20:07:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+replaying 2026-02-22T20:07:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:07:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:07:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:07:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:07:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+replaying '' '' 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:07:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: bfcca720-0a88-4106-99f7-e762d4fbade4 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790868,"replay_state":"idle"} 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:07:50 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:07:53' 2026-02-22T20:07:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:07:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:07:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:07:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790868,"replay_state":"idle"} 2026-02-22T20:07:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:07:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:07:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:07:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:07:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:07:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:07:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:07:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:07:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:07:50' 2026-02-22T20:07:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+replaying '2026-02-22 20:07:50' 2026-02-22T20:07:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:07:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:07:50' 2026-02-22T20:07:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:07:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:07:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:07:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK243244333mirror.0trial09819.2.3-689-gf90f4eaetrueOK43284339mirror.1trial09819.2.3-689-gf90f4eaefalseOK43354344mirror.2trial09819.2.3-689-gf90f4eaefalseOK43414346mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}43244333mirror.0trial0982026-02-22 20:07:24cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:07:53test_demote_imagebfcca720-0a88-4106-99f7-e762d4fbade4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790868,"replay_state":"idle"}43284339mirror.1trial0982026-02-22 20:07:50cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:07:53' 2026-02-22T20:07:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:07:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:07:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:07:50' 2026-02-22T20:07:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:07:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:07:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:07:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:07:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:07:54.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:54.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:54.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+stopped 2026-02-22T20:07:54.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:54.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:07:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:07:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:07:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:07:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:55.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+stopped '' '' 2026-02-22T20:07:55.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:07:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:07:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:07:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:07:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:07:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T20:07:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:07:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: bfcca720-0a88-4106-99f7-e762d4fbade4 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:07:53 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":136533.33,"bytes_per_snapshot":204800.0,"last_snapshot_bytes":409600,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790872,"remote_snapshot_timestamp":1771790872,"replay_state":"idle"} 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:07:54 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:07:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 173 .mirror.primary.bfcca720-0a88-4106-99f7-e762d4fbade4.f0091f57-fb40-45f2-a0c6-30f4142af857 (peer_uuids:[]) 2026-02-22T20:07:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 174 .mirror.primary.bfcca720-0a88-4106-99f7-e762d4fbade4.cea7bcc2-2151-49c6-b8b4-56fe33c7e1c7 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:07:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:07:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:07:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:07:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:07:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:07:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:07:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:07:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:07:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:07:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:07:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:07:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:07:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:07:53' 2026-02-22T20:07:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+stopped '2026-02-22 20:07:53' 2026-02-22T20:07:55.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:55.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:55.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:55.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:07:55.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:07:53' 2026-02-22T20:07:55.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:07:55.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:07:55.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:07:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK244434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:07:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:07:54test_demote_imagebfcca720-0a88-4106-99f7-e762d4fbade4up+stoppedlocal image is primary44474457mirror.1trial0982026-02-22 20:07:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":136533.33,"bytes_per_snapshot":204800.0,"last_snapshot_bytes":409600,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790872,"remote_snapshot_timestamp":1771790872,"replay_state":"idle"}2026-02-22 20:07:54' 2026-02-22T20:07:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:07:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:07:55.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:07:53' 2026-02-22T20:07:55.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:07:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:07:55.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:07:55.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:07:55.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:07:55.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:55.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:07:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster1 2026-02-22T20:07:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:07:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:07:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:0 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:07:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:07:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:07:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=21670 2026-02-22T20:07:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 21670 ']' 2026-02-22T20:07:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 21670 2026-02-22T20:07:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:07:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21670 '$2 == pid {print; exit 1}' 2026-02-22T20:07:55.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 21670 0.3 0.0 848896 59988 ? Ssl 19:51 0:03 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:07:55.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:55.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21670 '$2 == pid {print; exit 1}' 2026-02-22T20:07:56.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:07:56.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:56.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21670 '$2 == pid {print; exit 1}' 2026-02-22T20:07:56.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:07:56.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:07:56.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:56.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:56.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:07:56.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:07:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:07:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:07:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:07:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:07:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:0 2026-02-22T20:07:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:07:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:07:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:07:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:07:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:07:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:07:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:07:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:07:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:07:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:07:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:07:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T20:07:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:07:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:07:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:07:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T20:07:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:1 2026-02-22T20:07:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:07:56.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:07:56.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:07:56.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=21698 2026-02-22T20:07:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 21698 ']' 2026-02-22T20:07:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 21698 2026-02-22T20:07:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:07:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21698 '$2 == pid {print; exit 1}' 2026-02-22T20:07:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 21698 0.1 0.0 562032 57444 ? Ssl 19:51 0:01 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:07:56.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:56.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:57.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:57.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21698 '$2 == pid {print; exit 1}' 2026-02-22T20:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21698 '$2 == pid {print; exit 1}' 2026-02-22T20:07:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:07:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:07:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:07:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:07:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:07:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T20:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:57.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:07:57.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:07:57.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:07:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:1 2026-02-22T20:07:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:07:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:07:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:07:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:57.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:57.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:07:57.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:07:57.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:07:57.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:57.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:07:57.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:07:57.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:07:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:07:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:07:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T20:07:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:07:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:07:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:07:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T20:07:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:2 2026-02-22T20:07:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:07:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:07:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:07:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:07:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:07:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:07:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:07:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:07:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:07:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:07:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:07:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:07:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=21742 2026-02-22T20:07:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 21742 ']' 2026-02-22T20:07:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 21742 2026-02-22T20:07:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:07:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21742 '$2 == pid {print; exit 1}' 2026-02-22T20:07:57.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 21742 0.1 0.0 766940 52444 ? Ssl 19:51 0:01 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:07:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21742 '$2 == pid {print; exit 1}' 2026-02-22T20:07:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:07:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21742 '$2 == pid {print; exit 1}' 2026-02-22T20:07:58.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:07:58.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:07:58.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:58.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:58.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:07:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:07:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:07:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:07:58.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T20:07:58.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:07:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:07:58.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:07:58.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:07:58.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:07:58.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:07:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:07:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:07:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:2 2026-02-22T20:07:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:07:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:07:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:07:58.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:07:58.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:07:58.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:07:58.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T20:07:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:07:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:07:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:07:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T20:07:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:3 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:07:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:07:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:07:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:07:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:07:58.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=21802 2026-02-22T20:07:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 21802 ']' 2026-02-22T20:07:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 21802 2026-02-22T20:07:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:07:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21802 '$2 == pid {print; exit 1}' 2026-02-22T20:07:58.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 21802 0.1 0.0 766936 52436 ? Ssl 19:51 0:01 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:07:58.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:07:58.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:07:59.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:59.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21802 '$2 == pid {print; exit 1}' 2026-02-22T20:07:59.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:07:59.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:07:59.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=21802 '$2 == pid {print; exit 1}' 2026-02-22T20:07:59.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:07:59.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:07:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:07:59.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:07:59.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:07:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:07:59.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:07:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:07:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:07:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:07:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:07:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:07:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:07:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:07:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:07:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:3 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:07:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:07:59.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:07:59.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:07:59.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:07:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:07:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:07:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-22T20:07:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:07:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:07:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:07:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:07:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:07:59.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:07:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:07:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.57 bytes/sec: 33 MiB/s 2026-02-22T20:07:59.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test_demote_image 2026-02-22T20:07:59.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:07:59.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:07:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:07:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_demote_image 2026-02-22T20:08:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:08:01.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster1 2026-02-22T20:08:01.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:01.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:08:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:0 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:08:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:08:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:08:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:1 2026-02-22T20:08:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:08:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:08:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:08:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T20:08:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:2 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T20:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T20:08:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:08:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T20:08:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:08:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:3 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T20:08:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T20:08:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:08:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T20:08:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:08:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T20:08:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:08:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T20:08:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-22T20:08:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:08:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-22T20:08:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-22T20:08:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:08:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:08:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:08:01.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:08:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=175 2026-02-22T20:08:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:08:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:08:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:08:01.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:01.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:01.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:01.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:01.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:08:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:01.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:08:01.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=174 2026-02-22T20:08:01.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 175 = 174 2026-02-22T20:08:01.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:08:01.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:08:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:02.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:08:02.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=174 2026-02-22T20:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 175 = 174 2026-02-22T20:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.8 2026-02-22T20:08:02.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:02.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:02.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:02.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:02.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:02.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:08:02.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:08:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=174 2026-02-22T20:08:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 175 = 174 2026-02-22T20:08:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:08:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1.6 2026-02-22T20:08:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:08:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:08:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=174 2026-02-22T20:08:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 175 = 174 2026-02-22T20:08:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:08:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:08:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:08:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:08:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:06.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:08:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=175 2026-02-22T20:08:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 175 = 175 2026-02-22T20:08:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+unknown 2026-02-22T20:08:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-02-22T20:08:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T20:08:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:08:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: bfcca720-0a88-4106-99f7-e762d4fbade4 2026-02-22T20:08:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:08:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:03 2026-02-22T20:08:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:08:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:08:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:06' 2026-02-22T20:08:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:08:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:08:07.594 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:07.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:08:07.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:07.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:07.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:07.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:08:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:03' 2026-02-22T20:08:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+unknown '2026-02-22 20:08:03' 2026-02-22T20:08:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:03' 2026-02-22T20:08:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING1144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:08:03cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:08:05test_demote_imagebfcca720-0a88-4106-99f7-e762d4fbade4up+unknownremote image is not primary44474457mirror.1trial0982026-02-22 20:08:03cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image demoted2026-02-22 20:08:06' 2026-02-22T20:08:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:07.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:08:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:03' 2026-02-22T20:08:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:08:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:08:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:08:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+unknown 2026-02-22T20:08:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T20:08:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:08:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: bfcca720-0a88-4106-99f7-e762d4fbade4 2026-02-22T20:08:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:08:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:08:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:06 2026-02-22T20:08:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:08:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:08' 2026-02-22T20:08:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:08:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:08:08.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:08.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image demoted 2026-02-22T20:08:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:08:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:06' 2026-02-22T20:08:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+unknown '2026-02-22 20:08:06' 2026-02-22T20:08:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:06' 2026-02-22T20:08:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING1156315640mirror.0trial09819.2.3-689-gf90f4eaetrueOK56355647mirror.1trial09819.2.3-689-gf90f4eaefalseOK56425652mirror.2trial09819.2.3-689-gf90f4eaefalseOK56495654mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}56315640mirror.0trial0982026-02-22 20:08:05cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:08:08test_demote_imagebfcca720-0a88-4106-99f7-e762d4fbade4up+unknownremote image demoted56355647mirror.1trial0982026-02-22 20:08:06cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:08:08' 2026-02-22T20:08:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:08:08.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:06' 2026-02-22T20:08:08.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:08:08.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:08:08.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:08.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:08:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test_demote_image 2026-02-22T20:08:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:08:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_demote_image.export 2026-02-22T20:08:08.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_demote_image.export 2026-02-22T20:08:08.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export 2026-02-22T20:08:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_demote_image.export 2026-02-22T20:08:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_demote_image.export 2026-02-22T20:08:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:08:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:08:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test_demote_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export 2026-02-22T20:08:08.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:08:08.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:08:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:08:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:08:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:08:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test_demote_image 2026-02-22T20:08:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:08:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_demote_image 2026-02-22T20:08:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:08:09.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test_demote_image 2026-02-22T20:08:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T20:08:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:08:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:08:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T20:08:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:08:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:08:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:08:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:08:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T20:08:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:08:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:08:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:08:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:08:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:08:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:08:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:08:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:08:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:08:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:10.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:10.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:08:10.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:08:10.855 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:08:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:08:10.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:08:10.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:10.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T20:08:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:08:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:08:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:08:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:08:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:08:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:08:10.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:08:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:08:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:08:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:08:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:08:10.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:10.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:08:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T20:08:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:08:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:08:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:08:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T20:08:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:08:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:08:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:08:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:08:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:08:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:08:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:08:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:08:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:08:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:08:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:08:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:08:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:08:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:08:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T20:08:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:08:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:08:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:08:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:08:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:08:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:08:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:08:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:08:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:08:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:08:13.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:08:13.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:08:13.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+stopped 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+stopped '' '' 2026-02-22T20:08:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:14.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:14.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:14.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:14.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:14.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:14.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T20:08:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:08:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: bfcca720-0a88-4106-99f7-e762d4fbade4 2026-02-22T20:08:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:11 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790889,"replay_state":"idle"} 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:13 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 181 .mirror.primary.bfcca720-0a88-4106-99f7-e762d4fbade4.d2e2718f-61f0-44b9-b46d-596c3d73489d (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:08:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:08:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:08:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:08:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:08:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:08:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:14.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:08:14.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:14.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:11' 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+stopped '2026-02-22 20:08:11' 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:11' 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK256315640mirror.0trial09819.2.3-689-gf90f4eaetrueOK56355647mirror.1trial09819.2.3-689-gf90f4eaefalseOK56425652mirror.2trial09819.2.3-689-gf90f4eaefalseOK56495654mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}56315640mirror.0trial0982026-02-22 20:08:05cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:08:13test_demote_imagebfcca720-0a88-4106-99f7-e762d4fbade4up+stoppedlocal image is primary56355647mirror.1trial0982026-02-22 20:08:11cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790889,"replay_state":"idle"}2026-02-22 20:08:13' 2026-02-22T20:08:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:08:14.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:11' 2026-02-22T20:08:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:08:14.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:08:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:08:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:08:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+replaying 2026-02-22T20:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:14.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:14.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+replaying '' '' 2026-02-22T20:08:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T20:08:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:08:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: bfcca720-0a88-4106-99f7-e762d4fbade4 2026-02-22T20:08:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790889,"replay_state":"idle"} 2026-02-22T20:08:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:08:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:13 2026-02-22T20:08:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:08:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:11' 2026-02-22T20:08:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:08:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:08:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790889,"replay_state":"idle"} 2026-02-22T20:08:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:08:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:08:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:15.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:08:15.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:15.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:13' 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+replaying '2026-02-22 20:08:13' 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:13' 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK244434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:08:13cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:08:05test_demote_imagebfcca720-0a88-4106-99f7-e762d4fbade4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790889,"replay_state":"idle"}44474457mirror.1trial0982026-02-22 20:08:13cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:08:11' 2026-02-22T20:08:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:08:15.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:13' 2026-02-22T20:08:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:08:15.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:08:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:08:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:08:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:08:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster1 mirror test_demote_image 2026-02-22T20:08:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:08:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster1 mirror test_demote_image 2026-02-22T20:08:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:08:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_demote_image 2026-02-22T20:08:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 rm mirror/test_demote_image 2026-02-22T20:08:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:08:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:08:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: request image resync when remote is not primary' 2026-02-22T20:08:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:08:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:08:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:08:15 TEST: request image resync when remote is not primary 2026-02-22T20:08:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:08:15 TEST: request image resync when remote is not primary 2026-02-22T20:08:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_resync_image=test_resync_image 2026-02-22T20:08:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-22T20:08:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test_resync_image 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:08:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test_resync_image 2026-02-22T20:08:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:08:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:08:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-22T20:08:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:15.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:15.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:15.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:08:15.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_resync_image snapshot 2026-02-22T20:08:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:08:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_resync_image 2026-02-22T20:08:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_resync_image': 2026-02-22T20:08:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 170d59cacdb8 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.170d59cacdb8 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:08:15 2026 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:08:15 2026 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:08:15 2026 2026-02-22T20:08:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:08:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:08:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:08:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-22T20:08:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:08:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:08:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:08:16.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:08:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:08:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:08:16.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 12500.4 bytes/sec: 49 MiB/s 2026-02-22T20:08:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_resync_image 2026-02-22T20:08:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-22T20:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-22T20:08:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:08:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:08:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:08:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:08:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:08:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:08:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:08:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:08:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:08:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:08:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:08:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:08:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:08:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:08:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:08:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:08:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:08:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:08:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:2 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:08:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:08:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:08:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:08:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:08:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:08:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:17.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:08:17.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:08:17.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:08:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_resync_image 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_resync_image started 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_resync_image started 2026-02-22T20:08:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:08:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:08:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:08:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:08:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:08:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:08:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:08:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:08:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:08:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:08:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:08:18.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:18.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:18.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:08:18.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:18.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:08:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:08:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:08:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:08:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:08:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:18.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:18.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:18.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:08:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:08:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:08:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:08:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:08:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:08:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:18.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:18.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:18.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:08:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:08:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:08:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:08:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:08:18.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:08:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:08:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:08:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:08:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:18.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:08:18.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:08:18.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:08:18.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:08:18.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:08:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:08:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:08:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_resync_image 2026-02-22T20:08:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:08:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:08:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:08:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_resync_image 2026-02-22T20:08:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_resync_image 2026-02-22T20:08:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 178 2026-02-22T20:08:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_resync_image 2026-02-22T20:08:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:08:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_resync_image.status 2026-02-22T20:08:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_resync_image.status 2026-02-22T20:08:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.status 2026-02-22T20:08:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_resync_image.status 2026-02-22T20:08:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_resync_image.status 2026-02-22T20:08:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_resync_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.status 2026-02-22T20:08:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.status 2026-02-22T20:08:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_resync_image --format xml 2026-02-22T20:08:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:08:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=178 2026-02-22T20:08:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:08:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:08:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:08:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T20:08:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:19.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:08:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=177 2026-02-22T20:08:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 178 = 177 2026-02-22T20:08:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:08:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:08:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T20:08:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:19.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:08:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=178 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 178 = 178 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_resync_image 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+replaying 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:20.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+replaying '' '' 2026-02-22T20:08:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:20.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:08:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790896,"replay_state":"idle"} 2026-02-22T20:08:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.2 on trial098 2026-02-22T20:08:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:17 2026-02-22T20:08:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:08:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:18' 2026-02-22T20:08:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:08:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:08:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790896,"replay_state":"idle"} 2026-02-22T20:08:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:08:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:08:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:21.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.2 on trial098 2026-02-22T20:08:21.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:21.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:17' 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+replaying '2026-02-22 20:08:17' 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:17' 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:21.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:21.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK256315640mirror.0trial09819.2.3-689-gf90f4eaetrueOK56355647mirror.1trial09819.2.3-689-gf90f4eaefalseOK56425652mirror.2trial09819.2.3-689-gf90f4eaefalseOK56495654mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}56315640mirror.0trial0982026-02-22 20:08:05cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:08:18test_resync_image4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790896,"replay_state":"idle"}56425652mirror.2trial0982026-02-22 20:08:17cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:08:18' 2026-02-22T20:08:21.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:21.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:08:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:17' 2026-02-22T20:08:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:08:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:08:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:08:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:08:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:08:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+stopped 2026-02-22T20:08:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+stopped '' '' 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T20:08:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.2 on trial098 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:18 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790896,"replay_state":"idle"} 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:17 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 177 .mirror.primary.4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9.2dad0bcc-fae4-429e-b068-ce11b5ab6c5e (peer_uuids:[]) 2026-02-22T20:08:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 178 .mirror.primary.4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9.bd1a4f11-bf9e-4bb0-a765-40c8edb2f737 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:08:22.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:08:22.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:08:22.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:08:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:08:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:08:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.2 on trial098 2026-02-22T20:08:22.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:22.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:18' 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+stopped '2026-02-22 20:08:18' 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:18' 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK244434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:08:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:08:05test_resync_image4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9up+stoppedlocal image is primary44544468mirror.2trial0982026-02-22 20:08:18cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790896,"replay_state":"idle"}2026-02-22 20:08:17' 2026-02-22T20:08:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:08:22.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:18' 2026-02-22T20:08:22.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:08:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:08:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:08:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:08:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:08:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:08:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:08:22.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:08:22.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 6250.17 bytes/sec: 24 MiB/s 2026-02-22T20:08:22.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test_resync_image 2026-02-22T20:08:22.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:22.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:22.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:22.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_resync_image 2026-02-22T20:08:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ request_resync_image cluster1 mirror test_resync_image test_resync_image_id 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id_var_name=test_resync_image_id 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test_resync_image 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_resync_image 2026-02-22T20:08:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test_resync_image 2026-02-22T20:08:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:08:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'test_resync_image_id='\''1614ae0e1637'\''' 2026-02-22T20:08:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test_resync_image_id=1614ae0e1637 2026-02-22T20:08:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'test -n "$test_resync_image_id"' 2026-02-22T20:08:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -n 1614ae0e1637 2026-02-22T20:08:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test_resync_image 2026-02-22T20:08:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Flagged image for resync from primary 2026-02-22T20:08:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-22T20:08:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T20:08:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.2 on trial098 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:23 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:24' 2026-02-22T20:08:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:08:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:08:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:08:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T20:08:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:27.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:27.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:27.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:27.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T20:08:27.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.2 on trial098 2026-02-22T20:08:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:23 2026-02-22T20:08:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:08:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:27.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:26' 2026-02-22T20:08:27.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:08:27.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:08:27.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:27.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:08:31.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T20:08:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.2 on trial098 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:28 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:31' 2026-02-22T20:08:31.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:08:31.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:08:31.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T20:08:31.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:08:31.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:31.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:31.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.2 on trial098 2026-02-22T20:08:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:28' 2026-02-22T20:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+unknown '2026-02-22 20:08:28' 2026-02-22T20:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:28' 2026-02-22T20:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:31.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING1144434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:08:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:08:31test_resync_image4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9up+unknownremote image is not primary44544468mirror.2trial0982026-02-22 20:08:28cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:08:31' 2026-02-22T20:08:31.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:31.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:08:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:28' 2026-02-22T20:08:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:08:31.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:08:31.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:31.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:31.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:32.502 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T20:08:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:32.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:08:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.2 on trial098 2026-02-22T20:08:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:31 2026-02-22T20:08:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:08:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:28' 2026-02-22T20:08:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:08:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:08:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T20:08:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:08:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:32.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.2 on trial098 2026-02-22T20:08:32.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:32.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:31' 2026-02-22T20:08:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+unknown '2026-02-22 20:08:31' 2026-02-22T20:08:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:31' 2026-02-22T20:08:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING1156315640mirror.0trial09819.2.3-689-gf90f4eaetrueOK56355647mirror.1trial09819.2.3-689-gf90f4eaefalseOK56425652mirror.2trial09819.2.3-689-gf90f4eaefalseOK56495654mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}56315640mirror.0trial0982026-02-22 20:08:31cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:08:28test_resync_image4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9up+unknownremote image is not primary56425652mirror.2trial0982026-02-22 20:08:31cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:08:28' 2026-02-22T20:08:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:08:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:31' 2026-02-22T20:08:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:08:32.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:08:32.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:32.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:08:32.600 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test_resync_image 2026-02-22T20:08:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:08:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_resync_image 2026-02-22T20:08:32.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:08:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test_resync_image 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:08:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T20:08:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:08:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:08:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:08:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:08:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:08:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:08:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:08:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:08:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:08:33.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:08:33.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:08:33.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:33.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:33.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:08:33.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:33.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:33.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:08:33.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:08:33.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:08:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:33.945 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:33.946 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:33.947 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:08:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:08:33.948 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:08:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:34.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:34.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:2 2026-02-22T20:08:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:08:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:08:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:08:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:08:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:08:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T20:08:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:2 2026-02-22T20:08:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:08:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:2 2026-02-22T20:08:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T20:08:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:08:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:08:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:08:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:08:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:08:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:08:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:34.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:08:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:08:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:08:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:08:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:08:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:08:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:08:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+stopped 2026-02-22T20:08:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:35.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-22T20:08:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:08:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.2 on trial098 2026-02-22T20:08:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:31 2026-02-22T20:08:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:08:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790912,"replay_state":"idle"} 2026-02-22T20:08:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:33 2026-02-22T20:08:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:08:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 185 .mirror.primary.4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9.bdf501ab-d23b-4efa-9629-a9b60666553f (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:08:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:08:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:08:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:08:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-22T20:08:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:08:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:37.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.2 on trial098 2026-02-22T20:08:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:36 2026-02-22T20:08:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:08:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790912,"replay_state":"idle"} 2026-02-22T20:08:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:33 2026-02-22T20:08:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:08:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 185 .mirror.primary.4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9.bdf501ab-d23b-4efa-9629-a9b60666553f (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:08:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:08:37.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:08:37.157 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:37.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:08:37.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:37.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:08:37.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:37.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:08:37.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:37.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.2 on trial098 2026-02-22T20:08:37.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:37.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:36' 2026-02-22T20:08:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+stopped '2026-02-22 20:08:36' 2026-02-22T20:08:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:36' 2026-02-22T20:08:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:37.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK256315640mirror.0trial09819.2.3-689-gf90f4eaetrueOK56355647mirror.1trial09819.2.3-689-gf90f4eaefalseOK56425652mirror.2trial09819.2.3-689-gf90f4eaefalseOK56495654mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}56315640mirror.0trial0982026-02-22 20:08:31cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:08:33test_resync_image4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9up+stoppedlocal image is primary56425652mirror.2trial0982026-02-22 20:08:36cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790912,"replay_state":"idle"}2026-02-22 20:08:33' 2026-02-22T20:08:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:08:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:36' 2026-02-22T20:08:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:08:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:08:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:08:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:08:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:08:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+replaying 2026-02-22T20:08:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+replaying '' '' 2026-02-22T20:08:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T20:08:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790912,"replay_state":"idle"} 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.2 on trial098 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:33 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:36' 2026-02-22T20:08:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:08:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:08:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790912,"replay_state":"idle"} 2026-02-22T20:08:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:08:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:08:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.2 on trial098 2026-02-22T20:08:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:33' 2026-02-22T20:08:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+replaying '2026-02-22 20:08:33' 2026-02-22T20:08:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:33' 2026-02-22T20:08:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK244434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:08:38cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:08:31test_resync_image4d8e2a29-b3a1-4d71-b66f-4f0e3b5504e9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790912,"replay_state":"idle"}44544468mirror.2trial0982026-02-22 20:08:33cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:08:36' 2026-02-22T20:08:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:08:38.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:33' 2026-02-22T20:08:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:08:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:08:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:08:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:08:38.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:08:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test_resync_image 2026-02-22T20:08:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:08:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_resync_image.export 2026-02-22T20:08:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_resync_image.export 2026-02-22T20:08:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:38.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export 2026-02-22T20:08:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_resync_image.export 2026-02-22T20:08:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_resync_image.export 2026-02-22T20:08:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:08:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:08:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test_resync_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export 2026-02-22T20:08:38.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:08:38.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test_resync_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:08:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:08:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:08:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:08:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster1 mirror test_resync_image 2026-02-22T20:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster1 mirror test_resync_image 2026-02-22T20:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_resync_image 2026-02-22T20:08:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 rm mirror/test_resync_image 2026-02-22T20:08:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:08:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T20:08:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: request image resync when remote is not primary and daemon is offline' 2026-02-22T20:08:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:08:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:08:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:08:38 TEST: request image resync when remote is not primary and daemon is offline 2026-02-22T20:08:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:08:38 TEST: request image resync when remote is not primary and daemon is offline 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_resync_image=test_resync_image 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test_resync_image 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:08:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:08:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/test_resync_image 2026-02-22T20:08:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:08:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:08:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-22T20:08:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:08:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_resync_image snapshot 2026-02-22T20:08:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:08:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_resync_image 2026-02-22T20:08:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_resync_image': 2026-02-22T20:08:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:08:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:08:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:08:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 173fed54498e 2026-02-22T20:08:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.173fed54498e 2026-02-22T20:08:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:08:38 2026 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:08:38 2026 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:08:38 2026 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:08:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-22T20:08:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:08:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:08:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:08:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:08:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:08:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:08:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 6250.17 bytes/sec: 24 MiB/s 2026-02-22T20:08:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_resync_image 2026-02-22T20:08:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-22T20:08:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:08:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:08:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:08:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:08:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:08:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:08:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:40.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:40.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:40.420 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:40.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:40.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:40.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:40.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:40.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:40.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:08:40.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:08:40.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:40.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:08:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:08:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:08:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:08:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:08:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:2 2026-02-22T20:08:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:08:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:08:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:08:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:2 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:2 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T20:08:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:08:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:08:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:08:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:08:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:08:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:08:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:3 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:3 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T20:08:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:3 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:08:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T20:08:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:08:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:08:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:08:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T20:08:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:08:41.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:08:41.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:08:41.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:41.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:41.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:08:41.641 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:41.703 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:41.703 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:41.703 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:08:41.703 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:41.703 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:41.703 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:08:41.703 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:08:41.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:08:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:08:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:08:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:08:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:08:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:08:41.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:08:41.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:08:41.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:08:41.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:08:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:08:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:41.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:08:41.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:08:41.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:08:41.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:08:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_resync_image 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_resync_image started 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:08:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:08:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_resync_image started 2026-02-22T20:08:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:08:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:08:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:08:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:08:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:08:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:08:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:08:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:08:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:08:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:08:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:08:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:08:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:08:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:08:42.788 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:08:42.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:42.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:42.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:42.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:08:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:08:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_resync_image 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:08:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:08:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:08:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:08:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:08:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:08:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:08:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:08:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:08:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:08:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:08:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:08:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:08:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:08:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:08:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:08:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:08:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:08:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:08:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_resync_image 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_resync_image 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_resync_image 2026-02-22T20:08:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 182 2026-02-22T20:08:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_resync_image 2026-02-22T20:08:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:08:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_resync_image.status 2026-02-22T20:08:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_resync_image.status 2026-02-22T20:08:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.status 2026-02-22T20:08:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_resync_image.status 2026-02-22T20:08:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_resync_image.status 2026-02-22T20:08:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:08:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_resync_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.status 2026-02-22T20:08:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.status 2026-02-22T20:08:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_resync_image --format xml 2026-02-22T20:08:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:43.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:08:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=182 2026-02-22T20:08:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:08:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:08:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:08:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T20:08:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:08:43.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=181 2026-02-22T20:08:43.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 182 = 181 2026-02-22T20:08:43.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:08:43.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:08:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:43.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.status 2026-02-22T20:08:43.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T20:08:43.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:08:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=182 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 182 = 182 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_resync_image 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+replaying 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+replaying '' '' 2026-02-22T20:08:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:08:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:08:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790919,"replay_state":"idle"} 2026-02-22T20:08:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:08:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:40 2026-02-22T20:08:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:08:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:43' 2026-02-22T20:08:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:08:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:08:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:45.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790919,"replay_state":"idle"} 2026-02-22T20:08:45.026 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:45.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:08:45.027 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:45.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:08:45.028 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:45.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:08:45.030 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:40' 2026-02-22T20:08:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+replaying '2026-02-22 20:08:40' 2026-02-22T20:08:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:08:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:40' 2026-02-22T20:08:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK256315640mirror.0trial09819.2.3-689-gf90f4eaetrueOK56355647mirror.1trial09819.2.3-689-gf90f4eaefalseOK56425652mirror.2trial09819.2.3-689-gf90f4eaefalseOK56495654mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}56315640mirror.0trial0982026-02-22 20:08:31cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:08:43test_resync_image546c0a12-2569-4629-bd3d-c79ed0f4e0afup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790919,"replay_state":"idle"}56355647mirror.1trial0982026-02-22 20:08:40cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:08:43' 2026-02-22T20:08:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:08:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:40' 2026-02-22T20:08:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:08:45.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:08:45.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:08:45.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:08:45.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:08:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+stopped 2026-02-22T20:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+stopped '' '' 2026-02-22T20:08:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:08:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T20:08:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:43 2026-02-22T20:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790919,"replay_state":"idle"} 2026-02-22T20:08:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:40 2026-02-22T20:08:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:08:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 181 .mirror.primary.546c0a12-2569-4629-bd3d-c79ed0f4e0af.788591dd-2c1e-4633-bad2-29d3f5b8348a (peer_uuids:[]) 2026-02-22T20:08:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 182 .mirror.primary.546c0a12-2569-4629-bd3d-c79ed0f4e0af.2d723782-54fe-4bde-b620-069c25fbc784 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:08:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:08:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:08:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:08:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:08:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:08:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:08:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:08:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:43' 2026-02-22T20:08:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+stopped '2026-02-22 20:08:43' 2026-02-22T20:08:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:08:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:43' 2026-02-22T20:08:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK244434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:08:43cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:08:31test_resync_image546c0a12-2569-4629-bd3d-c79ed0f4e0afup+stoppedlocal image is primary44474457mirror.1trial0982026-02-22 20:08:43cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790919,"replay_state":"idle"}2026-02-22 20:08:40' 2026-02-22T20:08:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:08:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:43' 2026-02-22T20:08:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:08:46.371 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:08:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:08:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:08:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:08:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster1 2026-02-22T20:08:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:08:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:08:46.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:46.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T20:08:46.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:08:46.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:08:46.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:46.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:08:46.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T20:08:46.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:0 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:08:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:08:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:08:46.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:08:46.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:08:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=48275 2026-02-22T20:08:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 48275 ']' 2026-02-22T20:08:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 48275 2026-02-22T20:08:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:08:46.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:46.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48275 '$2 == pid {print; exit 1}' 2026-02-22T20:08:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 48275 0.3 0.0 766940 39492 ? Ssl 20:08 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48275 '$2 == pid {print; exit 1}' 2026-02-22T20:08:47.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:08:47.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:47.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48275 '$2 == pid {print; exit 1}' 2026-02-22T20:08:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:08:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:08:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:08:47.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:47.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:47.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:08:47.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:47.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:47.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T20:08:47.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:08:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:47.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:47.450 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:08:47.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:47.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:08:47.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:0 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:08:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:08:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:08:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:08:47.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:08:47.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:08:47.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:47.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T20:08:47.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:08:47.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:08:47.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:47.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:1 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:47.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:08:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:08:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:08:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:08:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:08:47.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:08:47.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:08:47.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=48303 2026-02-22T20:08:47.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 48303 ']' 2026-02-22T20:08:47.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 48303 2026-02-22T20:08:47.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:47.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:08:47.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:47.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48303 '$2 == pid {print; exit 1}' 2026-02-22T20:08:47.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 48303 0.4 0.0 766936 46108 ? Ssl 20:08 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:47.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:47.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48303 '$2 == pid {print; exit 1}' 2026-02-22T20:08:48.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:08:48.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:48.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48303 '$2 == pid {print; exit 1}' 2026-02-22T20:08:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:08:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:08:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:08:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:08:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:08:48.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:48.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:48.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:08:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:08:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:08:48.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:48.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:08:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:1 2026-02-22T20:08:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:08:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:08:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:08:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:08:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:08:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:08:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:08:48.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:08:48.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:08:48.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:48.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T20:08:48.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:08:48.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:08:48.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:48.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:08:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T20:08:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:2 2026-02-22T20:08:48.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:08:48.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:08:48.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:08:48.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:08:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=48347 2026-02-22T20:08:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 48347 ']' 2026-02-22T20:08:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 48347 2026-02-22T20:08:48.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:48.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:08:48.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:48.591 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48347 '$2 == pid {print; exit 1}' 2026-02-22T20:08:48.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 48347 0.4 0.0 766940 44516 ? Ssl 20:08 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:49.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:49.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48347 '$2 == pid {print; exit 1}' 2026-02-22T20:08:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:08:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48347 '$2 == pid {print; exit 1}' 2026-02-22T20:08:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:08:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:49.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:08:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:08:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:08:49.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T20:08:49.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:08:49.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:08:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:08:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:08:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:2 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:08:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:08:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:08:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:08:49.667 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:08:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:08:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T20:08:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:08:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:08:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:08:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T20:08:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:3 2026-02-22T20:08:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:08:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:08:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:08:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:08:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:08:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:08:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=48398 2026-02-22T20:08:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 48398 ']' 2026-02-22T20:08:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 48398 2026-02-22T20:08:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:08:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48398 '$2 == pid {print; exit 1}' 2026-02-22T20:08:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 48398 0.2 0.0 701372 37404 ? Ssl 20:08 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:08:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48398 '$2 == pid {print; exit 1}' 2026-02-22T20:08:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:08:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:08:50.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=48398 '$2 == pid {print; exit 1}' 2026-02-22T20:08:50.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:08:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:08:50.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:08:50.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:08:50.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:08:50.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:08:50.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:08:50.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:08:50.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:08:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T20:08:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:08:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:08:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:08:50.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:08:50.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:08:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:3 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:08:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:08:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:08:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:08:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:08:50.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:08:50.760 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:08:50.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-22T20:08:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:08:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:08:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:08:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:08:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:08:50.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:08:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 6250.17 bytes/sec: 24 MiB/s 2026-02-22T20:08:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test_resync_image 2026-02-22T20:08:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_resync_image 2026-02-22T20:08:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:08:52.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ request_resync_image cluster1 mirror test_resync_image test_resync_image_id 2026-02-22T20:08:52.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:52.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:52.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id_var_name=test_resync_image_id 2026-02-22T20:08:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test_resync_image 2026-02-22T20:08:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:08:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:08:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_resync_image 2026-02-22T20:08:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test_resync_image 2026-02-22T20:08:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:08:52.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'test_resync_image_id='\''160f5c1cfe5e'\''' 2026-02-22T20:08:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test_resync_image_id=160f5c1cfe5e 2026-02-22T20:08:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'test -n "$test_resync_image_id"' 2026-02-22T20:08:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -n 160f5c1cfe5e 2026-02-22T20:08:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test_resync_image 2026-02-22T20:08:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Flagged image for resync from primary 2026-02-22T20:08:52.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster1 2026-02-22T20:08:52.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:52.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:08:52.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:52.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:0 2026-02-22T20:08:52.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:08:52.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:08:52.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:08:52.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:52.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:1 2026-02-22T20:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T20:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T20:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T20:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T20:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:2 2026-02-22T20:08:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T20:08:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:08:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:08:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:3 2026-02-22T20:08:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:08:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:08:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:08:52.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:08:52.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:08:52.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-22T20:08:52.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T20:08:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:53.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:53 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:08:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:08:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:08:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:08:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:08:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T20:08:53.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:08:53.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:08:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:08:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:08:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:08:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:08:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:08:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:53' 2026-02-22T20:08:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+unknown '2026-02-22 20:08:53' 2026-02-22T20:08:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:08:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:08:53' 2026-02-22T20:08:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:08:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING244434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:08:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3faddown+unknownstatus not foundtest_resync_image546c0a12-2569-4629-bd3d-c79ed0f4e0afup+unknownremote image is not primary44474457mirror.1trial0982026-02-22 20:08:53cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3faddown+unknownstatus not found' 2026-02-22T20:08:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:08:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:08:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:08:53' 2026-02-22T20:08:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:08:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:08:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:08:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:08:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:53.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:53' 2026-02-22T20:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:08:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:08:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:08:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T20:08:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:08:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:08:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:08:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:08:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:08:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:08:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:08:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:08:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:08:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:08:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:08:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:08:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790923,"remote_snapshot_timestamp":1771790931,"replay_state":"idle"} 2026-02-22T20:08:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:56 2026-02-22T20:08:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:08:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:08:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:08:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:08:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:53' 2026-02-22T20:08:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:08:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:08:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:08:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:09:00.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T20:09:00.797 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:09:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:09:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:00.798 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:09:00.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image demoted 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:57 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:09:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:08:58' 2026-02-22T20:09:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:09:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:09:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T20:09:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:09:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 8 2026-02-22T20:09:08.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T20:09:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:09:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T20:09:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:09:08.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:09:08.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:09:07 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:09:08' 2026-02-22T20:09:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:09:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:09:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T20:09:08.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:09:08.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:09:08.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:09:08.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:09:08.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:09:08.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:09:08.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:09:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:09:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:09:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:07' 2026-02-22T20:09:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+unknown '2026-02-22 20:09:07' 2026-02-22T20:09:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:09:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:09:07' 2026-02-22T20:09:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:09:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING1157675775mirror.0trial09819.2.3-689-gf90f4eaetrueOK57715782mirror.1trial09819.2.3-689-gf90f4eaefalseOK57765785mirror.2trial09819.2.3-689-gf90f4eaefalseOK57815790mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}57675775mirror.0trial0982026-02-22 20:08:56cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:09:08test_resync_image546c0a12-2569-4629-bd3d-c79ed0f4e0afup+unknownremote image is not primary57715782mirror.1trial0982026-02-22 20:09:07cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:09:08' 2026-02-22T20:09:08.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:09:08.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:09:08.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:07' 2026-02-22T20:09:08.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:09:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:09:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:09:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:09:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:09:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test_resync_image 2026-02-22T20:09:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:09:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_resync_image 2026-02-22T20:09:09.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:09:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test_resync_image 2026-02-22T20:09:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T20:09:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:09:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:10.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T20:09:10.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:10.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:09:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:09:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:09:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T20:09:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:09:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:09:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:09:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:09:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:09:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:09:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:09:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T20:09:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:09:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:09:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:09:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:09:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:09:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:09:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:09:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:09:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:09:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:09:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:09:10.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:10.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:10.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:10.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:09:10.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:09:10.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:09:10.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:09:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:10.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:09:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:09:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:09:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:09:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:09:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:09:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:09:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:09:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:09:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:09:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:09:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:09:10.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:09:10.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:10.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:09:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T20:09:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:09:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:09:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:09:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T20:09:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:09:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:09:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:09:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:09:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:09:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:09:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:09:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T20:09:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:09:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:09:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:09:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:09:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:09:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:09:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:09:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T20:09:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:09:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:09:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T20:09:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:09:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:09:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:09:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:09:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T20:09:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:09:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:09:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:09:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:09:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:09:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:09:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:09:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+stopped 2026-02-22T20:09:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:09:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-22T20:09:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:09:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T20:09:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:09:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:09:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:09:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:09:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:09:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:09:12 2026-02-22T20:09:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:09:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:09:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:09:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790949,"replay_state":"idle"} 2026-02-22T20:09:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:09:13 2026-02-22T20:09:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:09:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 189 .mirror.primary.546c0a12-2569-4629-bd3d-c79ed0f4e0af.bf45c2f7-dac5-4a12-a99f-dfcaf3f2143e (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:09:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:09:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:09:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:09:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:09:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:09:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:09:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:09:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:09:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:09:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:09:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:09:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:09:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:12' 2026-02-22T20:09:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+stopped '2026-02-22 20:09:12' 2026-02-22T20:09:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:09:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:09:12' 2026-02-22T20:09:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:09:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK257675775mirror.0trial09819.2.3-689-gf90f4eaetrueOK57715782mirror.1trial09819.2.3-689-gf90f4eaefalseOK57765785mirror.2trial09819.2.3-689-gf90f4eaefalseOK57815790mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}57675775mirror.0trial0982026-02-22 20:08:56cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:09:13test_resync_image546c0a12-2569-4629-bd3d-c79ed0f4e0afup+stoppedlocal image is primary57715782mirror.1trial0982026-02-22 20:09:12cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790949,"replay_state":"idle"}2026-02-22 20:09:13' 2026-02-22T20:09:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:09:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:09:14.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:12' 2026-02-22T20:09:14.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:09:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:09:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:09:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:09:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:09:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+replaying 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+replaying '' '' 2026-02-22T20:09:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T20:09:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_resync_image: 2026-02-22T20:09:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 546c0a12-2569-4629-bd3d-c79ed0f4e0af 2026-02-22T20:09:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:09:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790949,"replay_state":"idle"} 2026-02-22T20:09:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:09:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:09:13 2026-02-22T20:09:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:09:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:09:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:09:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:09:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:09:12' 2026-02-22T20:09:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:09:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:09:15.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:09:15.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790949,"replay_state":"idle"} 2026-02-22T20:09:15.265 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:09:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:09:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:09:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:09:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:09:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:09:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:09:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:09:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:13' 2026-02-22T20:09:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+replaying '2026-02-22 20:09:13' 2026-02-22T20:09:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:09:13' 2026-02-22T20:09:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:09:15.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK244434452mirror.0trial09819.2.3-689-gf90f4eaetrueOK44474457mirror.1trial09819.2.3-689-gf90f4eaefalseOK44544468mirror.2trial09819.2.3-689-gf90f4eaefalseOK44614473mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest3bad19be-bf49-42ca-8b9d-5d734494ef44up+stoppedlocal image is primary44434452mirror.0trial0982026-02-22 20:09:13cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771790742,"remote_snapshot_timestamp":1771790742,"replay_state":"idle"}2026-02-22 20:08:56test_resync_image546c0a12-2569-4629-bd3d-c79ed0f4e0afup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790949,"replay_state":"idle"}44474457mirror.1trial0982026-02-22 20:09:13cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:09:12' 2026-02-22T20:09:15.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:09:15.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T20:09:15.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:13' 2026-02-22T20:09:15.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T20:09:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:09:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:09:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:09:15.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:09:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test_resync_image 2026-02-22T20:09:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:09:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_resync_image.export 2026-02-22T20:09:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_resync_image.export 2026-02-22T20:09:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export 2026-02-22T20:09:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_resync_image.export 2026-02-22T20:09:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_resync_image.export 2026-02-22T20:09:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:09:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:09:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test_resync_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export 2026-02-22T20:09:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:09:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test_resync_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:09:15.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:09:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:09:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_resync_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_resync_image.export 2026-02-22T20:09:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster1 mirror test_resync_image 2026-02-22T20:09:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster1 mirror test_resync_image 2026-02-22T20:09:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_resync_image 2026-02-22T20:09:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_resync_image 2026-02-22T20:09:15.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 rm mirror/test_resync_image 2026-02-22T20:09:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:09:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: disable mirror while daemon is stopped' 2026-02-22T20:09:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:09:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:09:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:09:15 TEST: disable mirror while daemon is stopped 2026-02-22T20:09:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:09:15 TEST: disable mirror while daemon is stopped 2026-02-22T20:09:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster1 2026-02-22T20:09:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:09:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T20:09:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:09:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:09:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:0 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:09:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:09:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:09:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:09:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:09:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:09:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:09:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=50538 2026-02-22T20:09:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 50538 ']' 2026-02-22T20:09:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 50538 2026-02-22T20:09:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50538 '$2 == pid {print; exit 1}' 2026-02-22T20:09:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 50538 0.4 0.0 766932 38008 ? Ssl 20:08 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:16.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50538 '$2 == pid {print; exit 1}' 2026-02-22T20:09:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:09:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50538 '$2 == pid {print; exit 1}' 2026-02-22T20:09:16.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:09:16.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:09:16.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:09:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:09:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:16.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:16.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:09:16.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:09:16.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:16.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:16.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:09:16.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:16.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T20:09:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:09:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:09:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:09:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:09:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:09:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:09:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:16.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:16.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:0 2026-02-22T20:09:16.970 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:16.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:16.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:16.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:09:16.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:09:16.981 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:09:16.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:16.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T20:09:16.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:09:16.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:16.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:16.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:09:16.983 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:1 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:09:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:09:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:09:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:09:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:09:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:09:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=50566 2026-02-22T20:09:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 50566 ']' 2026-02-22T20:09:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 50566 2026-02-22T20:09:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50566 '$2 == pid {print; exit 1}' 2026-02-22T20:09:17.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 50566 0.5 0.0 766940 41740 ? Ssl 20:08 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:17.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:17.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50566 '$2 == pid {print; exit 1}' 2026-02-22T20:09:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:09:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50566 '$2 == pid {print; exit 1}' 2026-02-22T20:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:09:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:09:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:09:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T20:09:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:09:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:09:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:09:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:09:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:1 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:09:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:09:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:09:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:09:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T20:09:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:09:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:09:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T20:09:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:2 2026-02-22T20:09:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:09:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:09:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:09:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:09:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=50594 2026-02-22T20:09:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 50594 ']' 2026-02-22T20:09:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 50594 2026-02-22T20:09:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50594 '$2 == pid {print; exit 1}' 2026-02-22T20:09:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 50594 0.3 0.0 701364 36188 ? Ssl 20:08 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50594 '$2 == pid {print; exit 1}' 2026-02-22T20:09:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:09:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50594 '$2 == pid {print; exit 1}' 2026-02-22T20:09:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:09:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:09:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:09:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:09:19.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:19.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:09:19.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:09:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:09:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:09:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:09:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:09:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:09:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:09:19.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:09:19.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:09:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:2 2026-02-22T20:09:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:09:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:09:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:09:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:09:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:09:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T20:09:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:09:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:09:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T20:09:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:3 2026-02-22T20:09:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:09:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:09:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:09:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:09:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:09:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:09:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=50654 2026-02-22T20:09:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 50654 ']' 2026-02-22T20:09:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 50654 2026-02-22T20:09:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50654 '$2 == pid {print; exit 1}' 2026-02-22T20:09:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 50654 0.3 0.0 701372 36668 ? Ssl 20:08 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50654 '$2 == pid {print; exit 1}' 2026-02-22T20:09:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:09:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=50654 '$2 == pid {print; exit 1}' 2026-02-22T20:09:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:09:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:09:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:09:20.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:20.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:09:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:09:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:09:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T20:09:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:09:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:09:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:09:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:09:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:09:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:3 2026-02-22T20:09:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:09:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:20.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:09:20.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:09:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:09:20.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster2 2026-02-22T20:09:20.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:20.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:20.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:09:20.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:20.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T20:09:20.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:0 2026-02-22T20:09:20.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:20.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:09:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster2:0 2026-02-22T20:09:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2:0 2026-02-22T20:09:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:09:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster2 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster2 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:09:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T20:09:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T20:09:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T20:09:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=25125 2026-02-22T20:09:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 25125 ']' 2026-02-22T20:09:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 25125 2026-02-22T20:09:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25125 '$2 == pid {print; exit 1}' 2026-02-22T20:09:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 25125 0.3 0.0 775128 56016 ? Ssl 19:53 0:02 rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25125 '$2 == pid {print; exit 1}' 2026-02-22T20:09:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:09:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25125 '$2 == pid {print; exit 1}' 2026-02-22T20:09:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:0 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:09:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:09:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:09:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:09:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:09:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:0 2026-02-22T20:09:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:09:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:09:21.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster2:0 2026-02-22T20:09:21.354 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T20:09:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T20:09:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T20:09:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster2:1 '' 2026-02-22T20:09:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:1 2026-02-22T20:09:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:09:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster2:1 2026-02-22T20:09:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2:1 2026-02-22T20:09:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:09:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster2 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster2 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:09:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T20:09:21.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T20:09:21.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T20:09:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=25153 2026-02-22T20:09:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 25153 ']' 2026-02-22T20:09:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 25153 2026-02-22T20:09:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:21.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:21.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25153 '$2 == pid {print; exit 1}' 2026-02-22T20:09:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 25153 0.1 0.0 766936 53364 ? Ssl 19:53 0:01 rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25153 '$2 == pid {print; exit 1}' 2026-02-22T20:09:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:09:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25153 '$2 == pid {print; exit 1}' 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:1 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:09:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:09:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:1 2026-02-22T20:09:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:09:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:09:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:09:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:09:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:09:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:09:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:09:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:09:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster2:1 2026-02-22T20:09:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:09:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T20:09:22.458 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T20:09:22.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T20:09:22.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:22.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster2:2 '' 2026-02-22T20:09:22.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:2 2026-02-22T20:09:22.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:22.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:22.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster2:2 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2:2 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:2 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster2 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster2 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:09:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T20:09:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T20:09:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T20:09:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=25196 2026-02-22T20:09:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 25196 ']' 2026-02-22T20:09:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 25196 2026-02-22T20:09:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25196 '$2 == pid {print; exit 1}' 2026-02-22T20:09:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 25196 0.1 0.0 766936 52196 ? Ssl 19:53 0:01 rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:23.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:23.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25196 '$2 == pid {print; exit 1}' 2026-02-22T20:09:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:09:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25196 '$2 == pid {print; exit 1}' 2026-02-22T20:09:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-02-22T20:09:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:2 2026-02-22T20:09:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:23.516 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:23.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:09:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:09:23.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:09:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T20:09:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:2 2026-02-22T20:09:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:09:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:09:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:09:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:09:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:09:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:09:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:09:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:09:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:09:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster2:2 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:2 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:09:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:23.543 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T20:09:23.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T20:09:23.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T20:09:23.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:23.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster2:3 '' 2026-02-22T20:09:23.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:3 2026-02-22T20:09:23.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:09:23.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:23.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster2:3 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2:3 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:3 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T20:09:23.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster2 2026-02-22T20:09:23.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster2 2026-02-22T20:09:23.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:09:23.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:09:23.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T20:09:23.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T20:09:23.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T20:09:23.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=25257 2026-02-22T20:09:23.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 25257 ']' 2026-02-22T20:09:23.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 25257 2026-02-22T20:09:23.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:23.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:23.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:23.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25257 '$2 == pid {print; exit 1}' 2026-02-22T20:09:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 25257 0.1 0.0 701368 49044 ? Ssl 19:53 0:01 rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:24.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:24.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25257 '$2 == pid {print; exit 1}' 2026-02-22T20:09:24.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:09:24.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:09:24.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=25257 '$2 == pid {print; exit 1}' 2026-02-22T20:09:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-02-22T20:09:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:3 2026-02-22T20:09:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T20:09:24.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:09:24.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:3 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T20:09:24.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:09:24.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:09:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster2:3 2026-02-22T20:09:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:3 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:09:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:09:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T20:09:24.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T20:09:24.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T20:09:24.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:09:24.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ disable_mirror cluster2 mirror test 2026-02-22T20:09:24.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:24.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:24.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:24.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test 2026-02-22T20:09:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring disabled 2026-02-22T20:09:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T20:09:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present 2026-02-22T20:09:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=10ed9072fff 2026-02-22T20:09:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 10ed9072fff 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z '' -o '' = 10ed9072fff 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = present 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster1 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:0 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:09:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:1 2026-02-22T20:09:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:09:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:09:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:09:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T20:09:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:09:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:09:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:09:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T20:09:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T20:09:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:09:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T20:09:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:2 2026-02-22T20:09:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:09:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:09:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:09:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T20:09:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:09:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:09:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:09:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T20:09:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T20:09:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:09:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T20:09:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:09:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:3 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:09:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T20:09:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:09:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:09:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-22T20:09:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:09:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id=10ed9072fff 2026-02-22T20:09:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:09:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 10ed9072fff 2026-02-22T20:09:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:09:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=10ed9072fff 2026-02-22T20:09:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=10ed9072fff 2026-02-22T20:09:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 10ed9072fff 2026-02-22T20:09:25.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 10ed9072fff -o 10ed9072fff = 10ed9072fff 2026-02-22T20:09:25.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:09:25.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = present 2026-02-22T20:09:25.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:25.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 10ed9072fff 2026-02-22T20:09:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=10ed9072fff 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=10ed9072fff 2026-02-22T20:09:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 10ed9072fff 2026-02-22T20:09:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 10ed9072fff -o 10ed9072fff = 10ed9072fff 2026-02-22T20:09:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:09:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = present 2026-02-22T20:09:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:09:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 10ed9072fff 2026-02-22T20:09:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=10ed9072fff 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=10ed9072fff 2026-02-22T20:09:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 10ed9072fff 2026-02-22T20:09:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 10ed9072fff -o 10ed9072fff = 10ed9072fff 2026-02-22T20:09:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:09:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = present 2026-02-22T20:09:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:09:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 10ed9072fff 2026-02-22T20:09:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:09:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=10ed9072fff 2026-02-22T20:09:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:09:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-22T20:09:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:09:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-22T20:09:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test': 2026-02-22T20:09:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:09:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:09:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:09:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 114815b984ce 2026-02-22T20:09:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.114815b984ce 2026-02-22T20:09:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:09:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:09:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:09:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:09:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 19:53:12 2026 2026-02-22T20:09:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 19:54:15 2026 2026-02-22T20:09:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 19:54:25 2026 2026-02-22T20:09:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:09:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:09:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: a5639638-93c3-4ec6-b3d3-5e442467aced 2026-02-22T20:09:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:09:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T20:09:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=present 2026-02-22T20:09:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:33.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T20:09:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:09:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T20:09:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:09:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = deleted 2026-02-22T20:09:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T20:09:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:09:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:09:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = deleted 2026-02-22T20:09:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:09:36.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T20:09:36.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:36.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=16d6cb4bf2d2 2026-02-22T20:09:36.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 16d6cb4bf2d2 2026-02-22T20:09:36.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z '' -o '' = 16d6cb4bf2d2 2026-02-22T20:09:36.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:09:36.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = present 2026-02-22T20:09:36.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:36.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:09:36.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:36.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:09:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:09:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:37.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:09:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:09:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:09:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:09:37.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:09:37.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:09:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:09:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:09:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:09:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:09:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:09:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:09:37.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:37.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:09:37.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:09:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:09:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:09:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:09:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:09:37.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:37.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:37.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:09:37.605 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:09:37.605 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:09:37.606 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:09:37.606 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:09:37.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:09:37.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:09:37.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:09:37.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:37.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: non-default namespace image mirroring' 2026-02-22T20:09:37.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:09:37.608 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:09:37.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:09:37 TEST: non-default namespace image mirroring 2026-02-22T20:09:37.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:09:37 TEST: non-default namespace image mirroring 2026-02-22T20:09:37.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog ' - replay' 2026-02-22T20:09:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:09:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:09:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:09:37 - replay 2026-02-22T20:09:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:09:37 - replay 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test snapshot 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror/ns1 test 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/ns1/test 2026-02-22T20:09:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-22T20:09:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:09:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror/ns1 test snapshot 2026-02-22T20:09:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:09:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test snapshot 2026-02-22T20:09:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:09:38.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test 2026-02-22T20:09:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test': 2026-02-22T20:09:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:09:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:09:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:09:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 17943e72d137 2026-02-22T20:09:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.17943e72d137 2026-02-22T20:09:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:09:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:09:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:09:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:09:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:09:37 2026 2026-02-22T20:09:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:09:37 2026 2026-02-22T20:09:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:09:37 2026 2026-02-22T20:09:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:09:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:09:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: f8910ebe-5b4a-460a-a9d2-b784d90cdbb6 2026-02-22T20:09:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:09:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns2 test snapshot 2026-02-22T20:09:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror/ns2 test 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/ns2/test 2026-02-22T20:09:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror/ns2 2026-02-22T20:09:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:09:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror/ns2 test snapshot 2026-02-22T20:09:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:09:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns2/test snapshot 2026-02-22T20:09:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:09:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/ns2/test 2026-02-22T20:09:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test': 2026-02-22T20:09:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:09:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:09:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:09:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 179d1c154062 2026-02-22T20:09:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.179d1c154062 2026-02-22T20:09:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:09:38 2026 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:09:38 2026 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:09:38 2026 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 69ae4c8d-bb52-4746-8daf-b997e41427ae 2026-02-22T20:09:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:09:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test 2026-02-22T20:09:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test started 2026-02-22T20:09:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:09:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror/ns1 test started 2026-02-22T20:09:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:09:40.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:09:40.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:40.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:40.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:09:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:09:40.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-22T20:09:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:09:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:09:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:09:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:09:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:09:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:09:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:09:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror/ns2 test 2026-02-22T20:09:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror/ns2 test started 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror/ns2 test started 2026-02-22T20:09:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns2/test 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:09:41.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:41.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:09:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:09:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:09:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns2/test 2026-02-22T20:09:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:09:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:09:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:09:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:09:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:09:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:09:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:41.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:41.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:41.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns2/test 2026-02-22T20:09:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:09:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:09:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:09:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:09:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:09:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:09:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:09:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror/ns1 test 100 2026-02-22T20:09:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:09:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:09:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:09:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:09:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.62 bytes/sec: 33 MiB/s 2026-02-22T20:09:41.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror/ns2 test 100 2026-02-22T20:09:41.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:09:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:09:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:09:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/ns2/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:09:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:09:41.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:09:41.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 12500.4 bytes/sec: 49 MiB/s 2026-02-22T20:09:41.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 test 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test 2026-02-22T20:09:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 188 2026-02-22T20:09:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 test 2026-02-22T20:09:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:09:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror/ns1-test.status 2026-02-22T20:09:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror/ns1-test.status 2026-02-22T20:09:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test.status 2026-02-22T20:09:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror/ns1-test.status 2026-02-22T20:09:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror/ns1-test.status 2026-02-22T20:09:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.status 2026-02-22T20:09:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test.status 2026-02-22T20:09:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test.status 2026-02-22T20:09:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test --format xml 2026-02-22T20:09:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:09:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=188 2026-02-22T20:09:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:09:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:09:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:09:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.status 2026-02-22T20:09:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.status 2026-02-22T20:09:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-22T20:09:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:42.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:09:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=186 2026-02-22T20:09:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 188 = 186 2026-02-22T20:09:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:09:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:09:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.status 2026-02-22T20:09:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.status 2026-02-22T20:09:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-22T20:09:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=188 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 188 = 188 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns2 test 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror/ns2 test 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns2/test 2026-02-22T20:09:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 189 2026-02-22T20:09:43.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns2 test 2026-02-22T20:09:43.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:09:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror/ns2-test.status 2026-02-22T20:09:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror/ns2-test.status 2026-02-22T20:09:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns2-test.status 2026-02-22T20:09:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror/ns2-test.status 2026-02-22T20:09:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror/ns2-test.status 2026-02-22T20:09:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:43.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.status 2026-02-22T20:09:43.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns2 test /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns2-test.status 2026-02-22T20:09:43.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:43.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:43.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:43.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns2-test.status 2026-02-22T20:09:43.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns2/test --format xml 2026-02-22T20:09:43.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:43.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:09:43.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=189 2026-02-22T20:09:43.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:09:43.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:09:43.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:09:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns2 test /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.status 2026-02-22T20:09:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.status 2026-02-22T20:09:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns2/test --format xml 2026-02-22T20:09:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:09:43.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=187 2026-02-22T20:09:43.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 189 = 187 2026-02-22T20:09:43.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:09:43.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:09:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns2 test /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.status 2026-02-22T20:09:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.status 2026-02-22T20:09:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns2/test --format xml 2026-02-22T20:09:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:09:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=189 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 189 = 189 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test up+replaying 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test up+replaying '' '' 2026-02-22T20:09:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test 2026-02-22T20:09:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:09:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: f8910ebe-5b4a-460a-a9d2-b784d90cdbb6 2026-02-22T20:09:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:09:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790978,"replay_state":"idle"} 2026-02-22T20:09:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:09:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:09:40 2026-02-22T20:09:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:09:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:09:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:09:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:09:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:09:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:09:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:09:45.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:09:45.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790978,"replay_state":"idle"} 2026-02-22T20:09:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:09:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:09:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:09:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:09:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:09:45.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:09:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:09:45.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:09:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:40' 2026-02-22T20:09:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test up+replaying '2026-02-22 20:09:40' 2026-02-22T20:09:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:09:40' 2026-02-22T20:09:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-02-22T20:09:45.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING158375846mirror.0trial09819.2.3-689-gf90f4eaetrueOK58415854mirror.1trial09819.2.3-689-gf90f4eaefalseOK58475859mirror.2trial09819.2.3-689-gf90f4eaefalseOK58515862mirror.3trial09819.2.3-689-gf90f4eaefalseOKtestf8910ebe-5b4a-460a-a9d2-b784d90cdbb6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790978,"replay_state":"idle"}58375846mirror.0trial0982026-02-22 20:09:40cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:09:45.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:09:45.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:09:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:40' 2026-02-22T20:09:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:09:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:09:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:09:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:09:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:09:45.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:45.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns2 test 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns2 test up+replaying 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:45.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:45.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror/ns2 test up+replaying '' '' 2026-02-22T20:09:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns2/test 2026-02-22T20:09:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:09:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 69ae4c8d-bb52-4746-8daf-b997e41427ae 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790979,"replay_state":"idle"} 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:09:41 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:09:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:09:46.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:09:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790979,"replay_state":"idle"} 2026-02-22T20:09:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:09:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:09:46.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:09:46.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:09:46.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:09:46.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:09:46.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:09:46.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:41' 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns2 test up+replaying '2026-02-22 20:09:41' 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:09:41' 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:46.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns2 --verbose --format xml 2026-02-22T20:09:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING158375846mirror.0trial09819.2.3-689-gf90f4eaetrueOK58415854mirror.1trial09819.2.3-689-gf90f4eaefalseOK58475859mirror.2trial09819.2.3-689-gf90f4eaefalseOK58515862mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest69ae4c8d-bb52-4746-8daf-b997e41427aeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790979,"replay_state":"idle"}58375846mirror.0trial0982026-02-22 20:09:41cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:09:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:09:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:09:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:41' 2026-02-22T20:09:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:09:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:09:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:09:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:09:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:09:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror/ns1 test 2026-02-22T20:09:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:09:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror/ns1-test.export 2026-02-22T20:09:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror/ns1-test.export 2026-02-22T20:09:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test.export 2026-02-22T20:09:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror/ns1-test.export 2026-02-22T20:09:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror/ns1-test.export 2026-02-22T20:09:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.export 2026-02-22T20:09:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.export 2026-02-22T20:09:46.194 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/ns1/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test.export 2026-02-22T20:09:46.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:09:46.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/ns1/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.export 2026-02-22T20:09:46.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:09:46.451 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.export 2026-02-22T20:09:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test.export 2026-02-22T20:09:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror/ns2 test 2026-02-22T20:09:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:09:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror/ns2-test.export 2026-02-22T20:09:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror/ns2-test.export 2026-02-22T20:09:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns2-test.export 2026-02-22T20:09:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror/ns2-test.export 2026-02-22T20:09:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror/ns2-test.export 2026-02-22T20:09:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.export 2026-02-22T20:09:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns2-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.export 2026-02-22T20:09:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/ns2/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns2-test.export 2026-02-22T20:09:46.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:09:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/ns2/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.export 2026-02-22T20:09:46.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:09:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns2-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.export 2026-02-22T20:09:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns2-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns2-test.export 2026-02-22T20:09:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog ' - disable mirroring / delete image' 2026-02-22T20:09:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:09:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:09:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:09:46 - disable mirroring / delete image 2026-02-22T20:09:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:09:46 - disable mirroring / delete image 2026-02-22T20:09:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror/ns1 test 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror/ns1 test 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test 2026-02-22T20:09:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test 2026-02-22T20:09:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:09:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ disable_mirror cluster2 mirror/ns2 test 2026-02-22T20:09:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/ns2/test 2026-02-22T20:09:48.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring disabled 2026-02-22T20:09:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror/ns1 test deleted 2026-02-22T20:09:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror/ns1 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-22T20:09:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T20:09:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:09:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror/ns1 test deleted '' 2026-02-22T20:09:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:48.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:09:48.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:48.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:48.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:48.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-22T20:09:48.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:48.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror/ns1 2026-02-22T20:09:48.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:48.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-22T20:09:48.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror/ns2 test deleted 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror/ns2 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-02-22T20:09:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T20:09:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror/ns2 test deleted '' 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror/ns2 2026-02-22T20:09:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-02-22T20:09:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:48.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:09:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:09:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror/ns2 test 2026-02-22T20:09:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror/ns2 test 2026-02-22T20:09:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns2 2026-02-22T20:09:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns2/test 2026-02-22T20:09:48.421 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/ns2/test 2026-02-22T20:09:48.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:09:48.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:48.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog ' - data pool' 2026-02-22T20:09:48.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:09:48.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:09:48.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:09:48 - data pool 2026-02-22T20:09:48.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:09:48 - data pool 2026-02-22T20:09:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ dp_image=test_data_pool 2026-02-22T20:09:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test_data_pool snapshot 128 --data-pool mirror_parent 2026-02-22T20:09:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror/ns1 test_data_pool 128 --data-pool mirror_parent 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:09:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 128 ']' 2026-02-22T20:09:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=128 2026-02-22T20:09:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:09:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering --data-pool mirror_parent mirror/ns1/test_data_pool 2026-02-22T20:09:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-22T20:09:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:09:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror/ns1 test_data_pool snapshot 2026-02-22T20:09:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:09:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test_data_pool snapshot 2026-02-22T20:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:09:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test_data_pool 2026-02-22T20:09:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_data_pool': 2026-02-22T20:09:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:09:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:09:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:09:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 17cd5678a5dd 2026-02-22T20:09:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: data_pool: mirror_parent 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.3.17cd5678a5dd 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, data-pool 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:09:48 2026 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:09:48 2026 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:09:48 2026 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 822a5892-837f-4390-9e15-f5c6d65df173 2026-02-22T20:09:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:09:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_data_pool cluster2 mirror/ns1 test_data_pool 2026-02-22T20:09:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:09:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror/ns1 2026-02-22T20:09:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_data_pool 2026-02-22T20:09:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 info mirror/ns1/test_data_pool 2026-02-22T20:09:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ data_pool=mirror_parent 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test mirror_parent = mirror_parent 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test_data_pool 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-02-22T20:09:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:09:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:09:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:09:50.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:09:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:09:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:09:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:09:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:09:50.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:09:50.144 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:09:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:09:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:09:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:09:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:09:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:09:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:09:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T20:09:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:09:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:09:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:09:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:09:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:09:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:09:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:09:50.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:09:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:09:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:09:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:09:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:09:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:09:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:09:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:09:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:09:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:09:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T20:09:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:09:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:09:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:09:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:09:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:09:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:09:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:09:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:09:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:09:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:09:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:09:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:09:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T20:09:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:09:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:09:50.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:09:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:09:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:09:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:09:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:09:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:09:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:09:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:50.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T20:09:50.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:09:50.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:09:50.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T20:09:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:09:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:09:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:09:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:09:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:09:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:09:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:09:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:09:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:09:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:09:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:09:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:09:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:09:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:09:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:09:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T20:09:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:09:51.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:09:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:09:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:09:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:09:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:09:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:09:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_data_pool cluster1 mirror/ns1 test_data_pool 2026-02-22T20:09:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror/ns1 2026-02-22T20:09:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_data_pool 2026-02-22T20:09:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 info mirror/ns1/test_data_pool 2026-02-22T20:09:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ data_pool=mirror_parent 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test mirror_parent = mirror_parent 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror/ns1 test_data_pool 100 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:09:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:09:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:09:51.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:09:51.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 12500.4 bytes/sec: 49 MiB/s 2026-02-22T20:09:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 test_data_pool 2026-02-22T20:09:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:09:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:09:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:09:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test_data_pool 2026-02-22T20:09:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test_data_pool 2026-02-22T20:09:52.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 17 2026-02-22T20:09:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 test_data_pool 2026-02-22T20:09:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:09:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror/ns1-test_data_pool.status 2026-02-22T20:09:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror/ns1-test_data_pool.status 2026-02-22T20:09:52.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T20:09:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror/ns1-test_data_pool.status 2026-02-22T20:09:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror/ns1-test_data_pool.status 2026-02-22T20:09:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T20:09:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test_data_pool /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T20:09:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T20:09:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-22T20:09:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:09:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=17 2026-02-22T20:09:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:09:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:09:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:09:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T20:09:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T20:09:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-22T20:09:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:52.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:09:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=16 2026-02-22T20:09:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 17 = 16 2026-02-22T20:09:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:09:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:09:52.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T20:09:52.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:52.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:52.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:52.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T20:09:52.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-22T20:09:52.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:09:52.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=16 2026-02-22T20:09:52.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 17 = 16 2026-02-22T20:09:52.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:09:52.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.8 2026-02-22T20:09:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T20:09:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T20:09:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-22T20:09:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:09:53.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=17 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 17 = 17 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test_data_pool 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:09:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:54.643 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying '' '' 2026-02-22T20:09:54.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:54.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:54.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:54.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:54.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:09:54.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:09:54.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:54.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:54.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test_data_pool 2026-02-22T20:09:54.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_data_pool: 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 822a5892-837f-4390-9e15-f5c6d65df173 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790989,"replay_state":"idle"} 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:09:51 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:09:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:09:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:09:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:09:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790989,"replay_state":"idle"} 2026-02-22T20:09:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:09:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:09:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:09:54.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:09:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:09:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:09:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:09:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:09:54.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:51' 2026-02-22T20:09:54.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test_data_pool up+replaying '2026-02-22 20:09:51' 2026-02-22T20:09:54.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:54.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:54.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:54.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:09:54.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:09:51' 2026-02-22T20:09:54.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:09:54.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:09:54.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-02-22T20:09:54.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING158375846mirror.0trial09819.2.3-689-gf90f4eaetrueOK58415854mirror.1trial09819.2.3-689-gf90f4eaefalseOK58475859mirror.2trial09819.2.3-689-gf90f4eaefalseOK58515862mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_data_pool822a5892-837f-4390-9e15-f5c6d65df173up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790989,"replay_state":"idle"}58375846mirror.0trial0982026-02-22 20:09:51cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:09:54.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:09:54.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-02-22T20:09:54.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:09:51' 2026-02-22T20:09:54.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-02-22T20:09:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:09:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:09:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:09:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:09:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror/ns1 test_data_pool 2026-02-22T20:09:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:09:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror/ns1-test_data_pool.export 2026-02-22T20:09:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror/ns1-test_data_pool.export 2026-02-22T20:09:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test_data_pool.export 2026-02-22T20:09:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror/ns1-test_data_pool.export 2026-02-22T20:09:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror/ns1-test_data_pool.export 2026-02-22T20:09:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:09:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T20:09:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T20:09:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/ns1/test_data_pool /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test_data_pool.export 2026-02-22T20:09:54.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:09:54.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/ns1/test_data_pool /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T20:09:54.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:09:55.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T20:09:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T20:09:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror/ns1 test_data_pool 2026-02-22T20:09:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:09:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:09:55.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror/ns1 test_data_pool 2026-02-22T20:09:55.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:09:55.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror/ns1 2026-02-22T20:09:55.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_data_pool 2026-02-22T20:09:55.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test_data_pool 2026-02-22T20:09:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test_data_pool 2026-02-22T20:09:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:09:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: simple image resync' 2026-02-22T20:09:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:09:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:09:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:09:56 TEST: simple image resync 2026-02-22T20:09:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:09:56 TEST: simple image resync 2026-02-22T20:09:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-22T20:09:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id_var_name=image_id 2026-02-22T20:09:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'image_id='\''16d6cb4bf2d2'\''' 2026-02-22T20:09:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ image_id=16d6cb4bf2d2 2026-02-22T20:09:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'test -n "$image_id"' 2026-02-22T20:09:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -n 16d6cb4bf2d2 2026-02-22T20:09:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-22T20:09:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Flagged image for resync from primary 2026-02-22T20:09:56.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test deleted 16d6cb4bf2d2 2026-02-22T20:09:56.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:09:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d6cb4bf2d2 2026-02-22T20:09:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 16d6cb4bf2d2 2026-02-22T20:09:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 16d6cb4bf2d2 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d6cb4bf2d2 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:56.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:09:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:09:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:09:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T20:09:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=present 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:56.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id= 2026-02-22T20:09:56.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:56.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:09:56.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T20:09:56.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:56.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:56.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:56.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:09:56.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:56.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:09:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = deleted 2026-02-22T20:09:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:09:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T20:09:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:09:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:09:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = deleted 2026-02-22T20:09:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:09:59.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:09:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:09:59.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:09:59.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:09:59.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:09:59.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:09:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:09:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:09:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = deleted 2026-02-22T20:09:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:09:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:10:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T20:10:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:10:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:10:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:10:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:10:03.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:03.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:03.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:03.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:03.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:03.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=16d62ed927dd 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 16d62ed927dd 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z '' -o '' = 16d62ed927dd 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = present 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:10:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:10:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:04.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:10:04.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:04.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:10:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:10:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:10:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:10:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:10:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:10:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:10:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:10:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:10:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:10:04.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:10:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:10:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:10:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:10:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:10:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:10:04.646 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:10:04.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:10:04.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:10:04.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:10:04.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:10:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:10:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:10:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T20:10:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:10:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:10:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:10:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: a5639638-93c3-4ec6-b3d3-5e442467aced 2026-02-22T20:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790973,"replay_state":"idle"} 2026-02-22T20:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:00 2026-02-22T20:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:10:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:10:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:10:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:10:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790973,"replay_state":"idle"} 2026-02-22T20:10:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:10:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:10:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:10:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:10:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:10:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:10:05.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:00' 2026-02-22T20:10:05.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:10:00' 2026-02-22T20:10:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:10:00' 2026-02-22T20:10:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:10:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING158375846mirror.0trial09819.2.3-689-gf90f4eaetrueOK58415854mirror.1trial09819.2.3-689-gf90f4eaefalseOK58475859mirror.2trial09819.2.3-689-gf90f4eaefalseOK58515862mirror.3trial09819.2.3-689-gf90f4eaefalseOKtesta5639638-93c3-4ec6-b3d3-5e442467acedup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790973,"replay_state":"idle"}58375846mirror.0trial0982026-02-22 20:10:00cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:10:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:10:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:10:05.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:00' 2026-02-22T20:10:05.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:10:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:10:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:10:05.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:05.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:05.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:05.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test 2026-02-22T20:10:05.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:05.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:05.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:10:05.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T20:10:05.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.export 2026-02-22T20:10:05.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:10:05.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T20:10:05.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T20:10:05.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.export 2026-02-22T20:10:05.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:10:05.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:05.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:05.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T20:10:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:10:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:05.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:10:05.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T20:10:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: image resync while replayer is stopped' 2026-02-22T20:10:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:10:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:10:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:10:06 TEST: image resync while replayer is stopped 2026-02-22T20:10:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:10:06 TEST: image resync while replayer is stopped 2026-02-22T20:10:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemons cluster1 rbd mirror stop mirror/test 2026-02-22T20:10:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_instance=cluster1 2026-02-22T20:10:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance=cluster1 2026-02-22T20:10:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loop_instance 2026-02-22T20:10:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:10:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:10:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror/test 2026-02-22T20:10:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:10:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:06.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:06.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:06.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:06.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror/test 2026-02-22T20:10:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T20:10:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:10:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:07.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:10:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:10:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:10:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:10:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:10:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:10:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:10:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:07.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:07.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:07.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:07.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:10:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:10:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:10:07.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:10:07.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:10:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:10:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-22T20:10:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id_var_name=image_id 2026-02-22T20:10:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'image_id='\''16d62ed927dd'\''' 2026-02-22T20:10:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ image_id=16d62ed927dd 2026-02-22T20:10:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'test -n "$image_id"' 2026-02-22T20:10:07.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -n 16d62ed927dd 2026-02-22T20:10:07.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-22T20:10:07.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Flagged image for resync from primary 2026-02-22T20:10:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-22T20:10:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_instance=cluster1 2026-02-22T20:10:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance=cluster1 2026-02-22T20:10:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loop_instance 2026-02-22T20:10:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:10:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:10:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-22T20:10:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:10:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T20:10:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:07.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-22T20:10:07.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:07.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test deleted 16d62ed927dd 2026-02-22T20:10:07.395 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:10:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d62ed927dd 2026-02-22T20:10:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 16d62ed927dd 2026-02-22T20:10:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:10:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:10:07.496 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 16d62ed927dd 2026-02-22T20:10:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:10:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d62ed927dd 2026-02-22T20:10:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:10:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:10:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:07.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:07.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:07.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:07.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:07.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id= 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_instance=cluster1 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance=cluster1 2026-02-22T20:10:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loop_instance 2026-02-22T20:10:07.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:07.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:07.523 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:10:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:10:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:07.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-22T20:10:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=present 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:07.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id=16d69714b8dd 2026-02-22T20:10:07.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:10:07.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:10:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present 16d69714b8dd 2026-02-22T20:10:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:10:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d69714b8dd 2026-02-22T20:10:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:10:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:10:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=16d69714b8dd 2026-02-22T20:10:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 16d69714b8dd 2026-02-22T20:10:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 16d69714b8dd -o 16d69714b8dd = 16d69714b8dd 2026-02-22T20:10:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:10:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = present 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:08.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:10:08.762 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:10:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:10:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:10:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:10:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:10:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:10:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:10:08.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:10:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:10:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:10:08.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:10:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:10:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:10:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:10:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T20:10:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:10:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:10:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:09.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:10:09.838 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:10:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:10:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: a5639638-93c3-4ec6-b3d3-5e442467aced 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790973,"replay_state":"idle"} 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:07 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:10:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:10:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:10:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:10:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790973,"replay_state":"idle"} 2026-02-22T20:10:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:10:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:10:09.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:09.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:10:09.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:10:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:10:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:10:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:07' 2026-02-22T20:10:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:10:07' 2026-02-22T20:10:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:10:07' 2026-02-22T20:10:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:10:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING158375846mirror.0trial09819.2.3-689-gf90f4eaetrueOK58415854mirror.1trial09819.2.3-689-gf90f4eaefalseOK58475859mirror.2trial09819.2.3-689-gf90f4eaefalseOK58515862mirror.3trial09819.2.3-689-gf90f4eaefalseOKtesta5639638-93c3-4ec6-b3d3-5e442467acedup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790973,"replay_state":"idle"}58375846mirror.0trial0982026-02-22 20:10:07cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:10:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:10:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:10:09.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:07' 2026-02-22T20:10:09.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:10:09.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:10:09.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:09.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:10:09.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:09.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:09.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:09.926 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test 2026-02-22T20:10:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:10:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T20:10:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.export 2026-02-22T20:10:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:10:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T20:10:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T20:10:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.export 2026-02-22T20:10:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:10:09.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:09.930 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:09.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T20:10:10.269 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:10:10.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:10:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:10.449 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: request image resync while daemon is offline' 2026-02-22T20:10:10.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:10:10.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:10:10.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:10:10 TEST: request image resync while daemon is offline 2026-02-22T20:10:10.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:10:10 TEST: request image resync while daemon is offline 2026-02-22T20:10:10.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster1 2026-02-22T20:10:10.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:10.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:10:10.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:10:10.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:10.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T20:10:10.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:10:10.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:10:10.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:10.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:0 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:10:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:10:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:10:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:10:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=51607 2026-02-22T20:10:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 51607 ']' 2026-02-22T20:10:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 51607 2026-02-22T20:10:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51607 '$2 == pid {print; exit 1}' 2026-02-22T20:10:10.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 51607 1.1 0.0 775128 51756 ? Ssl 20:09 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:10.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:10.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:11.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:11.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51607 '$2 == pid {print; exit 1}' 2026-02-22T20:10:11.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:10:11.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:11.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51607 '$2 == pid {print; exit 1}' 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:11.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:11.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:11.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:11.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:11.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:11.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:0 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:11.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:10:11.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:10:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:10:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T20:10:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:10:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:10:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:10:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T20:10:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:1 2026-02-22T20:10:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:10:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:10:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:10:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:10:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:10:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:10:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:10:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:10:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:10:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:10:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:10:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=51635 2026-02-22T20:10:11.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 51635 ']' 2026-02-22T20:10:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 51635 2026-02-22T20:10:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:11.599 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51635 '$2 == pid {print; exit 1}' 2026-02-22T20:10:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 51635 0.2 0.0 494420 37964 ? Ssl 20:09 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51635 '$2 == pid {print; exit 1}' 2026-02-22T20:10:12.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:10:12.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:12.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51635 '$2 == pid {print; exit 1}' 2026-02-22T20:10:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:10:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:10:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:10:12.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:12.656 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:10:12.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:12.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:1 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:10:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:10:12.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:12.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:12.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:10:12.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:12.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:10:12.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:10:12.681 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:10:12.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:12.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T20:10:12.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:10:12.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:10:12.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:12.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:10:12.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T20:10:12.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:2 2026-02-22T20:10:12.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:12.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:10:12.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:10:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:10:12.696 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:10:12.696 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:10:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=51663 2026-02-22T20:10:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 51663 ']' 2026-02-22T20:10:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 51663 2026-02-22T20:10:12.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:12.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:12.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51663 '$2 == pid {print; exit 1}' 2026-02-22T20:10:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 51663 0.2 0.0 699320 37276 ? Ssl 20:09 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51663 '$2 == pid {print; exit 1}' 2026-02-22T20:10:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:10:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51663 '$2 == pid {print; exit 1}' 2026-02-22T20:10:13.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:10:13.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:10:13.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:10:13.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:10:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T20:10:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:10:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:10:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:10:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:10:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:10:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:10:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:10:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:10:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:10:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:2 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:10:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:10:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:10:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:10:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T20:10:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:10:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:10:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:10:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T20:10:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:3 2026-02-22T20:10:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:10:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:10:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:10:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:10:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=51723 2026-02-22T20:10:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 51723 ']' 2026-02-22T20:10:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 51723 2026-02-22T20:10:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51723 '$2 == pid {print; exit 1}' 2026-02-22T20:10:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 51723 0.2 0.0 701368 37884 ? Ssl 20:09 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51723 '$2 == pid {print; exit 1}' 2026-02-22T20:10:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:10:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:10:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=51723 '$2 == pid {print; exit 1}' 2026-02-22T20:10:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:10:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:10:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:10:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:10:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:10:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:10:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:10:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:10:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:10:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:10:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:10:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T20:10:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:10:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:10:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:10:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:10:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:10:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:10:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:10:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:10:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:10:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:10:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:10:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:10:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:10:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:3 2026-02-22T20:10:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:10:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:10:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:10:14.870 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:10:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:10:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-22T20:10:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id_var_name=image_id 2026-02-22T20:10:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'image_id='\''16d69714b8dd'\''' 2026-02-22T20:10:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ image_id=16d69714b8dd 2026-02-22T20:10:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'test -n "$image_id"' 2026-02-22T20:10:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -n 16d69714b8dd 2026-02-22T20:10:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-22T20:10:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Flagged image for resync from primary 2026-02-22T20:10:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster1 2026-02-22T20:10:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:10:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:0 2026-02-22T20:10:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:10:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T20:10:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:1 2026-02-22T20:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:2 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:10:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T20:10:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:10:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:3 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T20:10:14.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:10:14.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:14.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T20:10:14.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:10:14.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:14.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test deleted 16d69714b8dd 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=deleted 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d69714b8dd 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 16d69714b8dd 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:10:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 16d69714b8dd 2026-02-22T20:10:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:10:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d69714b8dd 2026-02-22T20:10:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:10:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:10:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=16d69714b8dd 2026-02-22T20:10:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 16d69714b8dd 2026-02-22T20:10:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 16d69714b8dd -o 16d69714b8dd = 16d69714b8dd 2026-02-22T20:10:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:10:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = present 2026-02-22T20:10:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:10:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 16d69714b8dd 2026-02-22T20:10:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:10:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d69714b8dd 2026-02-22T20:10:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:10:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:10:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=16d69714b8dd 2026-02-22T20:10:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 16d69714b8dd 2026-02-22T20:10:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 16d69714b8dd -o 16d69714b8dd = 16d69714b8dd 2026-02-22T20:10:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:10:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = present 2026-02-22T20:10:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:10:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:10:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 16d69714b8dd 2026-02-22T20:10:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:10:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d69714b8dd 2026-02-22T20:10:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:10:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:10:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:18.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:18.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=16d69714b8dd 2026-02-22T20:10:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 16d69714b8dd 2026-02-22T20:10:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 16d69714b8dd -o 16d69714b8dd = 16d69714b8dd 2026-02-22T20:10:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:10:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = present 2026-02-22T20:10:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:10:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:10:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test deleted 16d69714b8dd 2026-02-22T20:10:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=deleted 2026-02-22T20:10:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=16d69714b8dd 2026-02-22T20:10:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:10:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:10:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=178c96e5f04c 2026-02-22T20:10:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 178c96e5f04c 2026-02-22T20:10:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 16d69714b8dd -o 16d69714b8dd = 178c96e5f04c 2026-02-22T20:10:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test deleted = deleted 2026-02-22T20:10:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T20:10:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=present 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id= 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:22.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image_id=178c96e5f04c 2026-02-22T20:10:22.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:10:22.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:10:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_present cluster1 mirror test present 178c96e5f04c 2026-02-22T20:10:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=present 2026-02-22T20:10:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id=178c96e5f04c 2026-02-22T20:10:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=deleted 2026-02-22T20:10:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_image_id 2026-02-22T20:10:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror test 2026-02-22T20:10:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test 2026-02-22T20:10:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T20:10:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_image_id=178c96e5f04c 2026-02-22T20:10:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 178c96e5f04c 2026-02-22T20:10:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -z 178c96e5f04c -o 178c96e5f04c = 178c96e5f04c 2026-02-22T20:10:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=present 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test present = present 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:10:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T20:10:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:10:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:10:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:10:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T20:10:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:10:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:10:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:10:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:10:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:10:23.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:10:23.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:10:23.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:23.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T20:10:23.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:10:23.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:23.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:23.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:23.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:10:23.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:10:23.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:10:23.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:10:23.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:23.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:23.422 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T20:10:23.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:10:23.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:10:23.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:10:23.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:10:23.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:10:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:10:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:10:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T20:10:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T20:10:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T20:10:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test: 2026-02-22T20:10:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: a5639638-93c3-4ec6-b3d3-5e442467aced 2026-02-22T20:10:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:10:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790973,"replay_state":"idle"} 2026-02-22T20:10:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:10:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:20 2026-02-22T20:10:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:10:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:10:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:10:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:10:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790973,"replay_state":"idle"} 2026-02-22T20:10:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:10:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:10:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:10:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:10:24.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:10:24.533 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:10:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:20' 2026-02-22T20:10:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 20:10:20' 2026-02-22T20:10:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:10:20' 2026-02-22T20:10:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:10:24.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING160166028mirror.0trial09819.2.3-689-gf90f4eaetrueOK60206025mirror.1trial09819.2.3-689-gf90f4eaefalseOK60306037mirror.2trial09819.2.3-689-gf90f4eaefalseOK60346042mirror.3trial09819.2.3-689-gf90f4eaefalseOKtesta5639638-93c3-4ec6-b3d3-5e442467acedup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771790973,"replay_state":"idle"}60166028mirror.0trial0982026-02-22 20:10:20cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:10:24.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:10:24.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T20:10:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:20' 2026-02-22T20:10:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T20:10:24.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:10:24.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:10:24.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:24.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test 2026-02-22T20:10:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:10:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T20:10:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test.export 2026-02-22T20:10:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:10:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T20:10:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T20:10:24.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test.export 2026-02-22T20:10:24.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:10:24.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export 2026-02-22T20:10:24.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:10:24.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T20:10:24.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test.export 2026-02-22T20:10:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test 2026-02-22T20:10:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test 2026-02-22T20:10:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test 2026-02-22T20:10:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-22T20:10:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-22T20:10:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:10:25.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:25.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:10:25.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: split-brain' 2026-02-22T20:10:25.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:10:25.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:10:25.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:10:25 TEST: split-brain 2026-02-22T20:10:25.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:10:25 TEST: split-brain 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ image=split-brain 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror split-brain snapshot 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror split-brain 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:10:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering mirror/split-brain 2026-02-22T20:10:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:10:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:10:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror split-brain snapshot 2026-02-22T20:10:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:10:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/split-brain snapshot 2026-02-22T20:10:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:10:26.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/split-brain 2026-02-22T20:10:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'split-brain': 2026-02-22T20:10:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 128 MiB in 32 objects 2026-02-22T20:10:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:10:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:10:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 180aac1c6cf3 2026-02-22T20:10:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.180aac1c6cf3 2026-02-22T20:10:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:10:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:10:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:10:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:10:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:10:25 2026 2026-02-22T20:10:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:10:25 2026 2026-02-22T20:10:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:10:25 2026 2026-02-22T20:10:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:10:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:10:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: effb6c3c-068a-49cd-a98d-31ad151c6b79 2026-02-22T20:10:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:10:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-02-22T20:10:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:10:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying 2026-02-22T20:10:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T20:10:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T20:10:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image split-brain: (2) No such file or directory 2026-02-22T20:10:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status= 2026-02-22T20:10:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:10:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:10:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:27.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T20:10:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='split-brain: 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: effb6c3c-068a-49cd-a98d-31ad151c6b79 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791026,"replay_state":"idle"} 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:28 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:10:29.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:10:29.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:10:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:10:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791026,"replay_state":"idle"} 2026-02-22T20:10:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:10:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:10:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:10:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:10:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:10:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:28' 2026-02-22T20:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-02-22 20:10:28' 2026-02-22T20:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:10:28' 2026-02-22T20:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:10:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING160166028mirror.0trial09819.2.3-689-gf90f4eaetrueOK60206025mirror.1trial09819.2.3-689-gf90f4eaefalseOK60306037mirror.2trial09819.2.3-689-gf90f4eaefalseOK60346042mirror.3trial09819.2.3-689-gf90f4eaefalseOKsplit-braineffb6c3c-068a-49cd-a98d-31ad151c6b79up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791026,"replay_state":"idle"}60206025mirror.1trial0982026-02-22 20:10:28cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:10:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:10:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T20:10:29.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:28' 2026-02-22T20:10:29.428 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T20:10:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:10:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:10:29.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:29.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:29.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:29.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror split-brain --force 2026-02-22T20:10:29.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force=--force 2026-02-22T20:10:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/split-brain --force 2026-02-22T20:10:33.862 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror split-brain 2026-02-22T20:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror split-brain stopped 2026-02-22T20:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror split-brain stopped 2026-02-22T20:10:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:10:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:10:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:10:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/split-brain 2026-02-22T20:10:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:10:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:10:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:10:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:10:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:10:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:10:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:10:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/split-brain 2026-02-22T20:10:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:10:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:10:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:10:34.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:10:34.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:10:34.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:34.887 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:34.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:34.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:34.888 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/split-brain 2026-02-22T20:10:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/split-brain'] 2026-02-22T20:10:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:10:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:10:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/split-brain 2026-02-22T20:10:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:10:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:10:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:10:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:10:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:10:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:10:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:10:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:10:34.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:34.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:34.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:34.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/split-brain 2026-02-22T20:10:35.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:10:35.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:10:35.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:10:35.008 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:10:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:10:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+stopped 2026-02-22T20:10:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:10:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:36.011 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+stopped '' '' 2026-02-22T20:10:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:10:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T20:10:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='split-brain: 2026-02-22T20:10:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: effb6c3c-068a-49cd-a98d-31ad151c6b79 2026-02-22T20:10:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:10:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:10:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:10:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:35 2026-02-22T20:10:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22T20:10:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:10:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 200 .mirror.primary.effb6c3c-068a-49cd-a98d-31ad151c6b79.3a7652ad-9fe3-4571-84b5-b211964ee62d (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:10:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:10:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:10:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:10:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:10:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:10:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:10:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:10:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:10:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:10:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:10:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:10:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:10:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:35' 2026-02-22T20:10:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+stopped '2026-02-22 20:10:35' 2026-02-22T20:10:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:10:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:10:35' 2026-02-22T20:10:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:10:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING160166028mirror.0trial09819.2.3-689-gf90f4eaetrueOK60206025mirror.1trial09819.2.3-689-gf90f4eaefalseOK60306037mirror.2trial09819.2.3-689-gf90f4eaefalseOK60346042mirror.3trial09819.2.3-689-gf90f4eaefalseOKsplit-braineffb6c3c-068a-49cd-a98d-31ad151c6b79up+stoppedlocal image is primary60206025mirror.1trial0982026-02-22 20:10:35cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:10:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:10:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T20:10:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:35' 2026-02-22T20:10:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T20:10:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:10:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:10:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:10:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:10:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster1 mirror split-brain 10 2026-02-22T20:10:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=10 2026-02-22T20:10:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:10:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:10:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 bench mirror/split-brain --io-type write --io-size 4096 --io-threads 1 --io-total 40960 --io-pattern rand 2026-02-22T20:10:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 40960 pattern random 2026-02-22T20:10:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:10:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 10 ops/sec: 2500.08 bytes/sec: 9.8 MiB/s 2026-02-22T20:10:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster1 mirror split-brain 2026-02-22T20:10:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/split-brain 2026-02-22T20:10:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:10:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+error split-brain 2026-02-22T20:10:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+error 2026-02-22T20:10:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern=split-brain 2026-02-22T20:10:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:37.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-22T20:10:37.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:37.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:37.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:37.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+error 2026-02-22T20:10:37.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern=split-brain 2026-02-22T20:10:37.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:37.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:37.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:37.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T20:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='split-brain: 2026-02-22T20:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: effb6c3c-068a-49cd-a98d-31ad151c6b79 2026-02-22T20:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:10:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:35 2026-02-22T20:10:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:10:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+error' 2026-02-22T20:10:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:10:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:10:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-22T20:10:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+error 2026-02-22T20:10:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern=split-brain 2026-02-22T20:10:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T20:10:39.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='split-brain: 2026-02-22T20:10:39.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: effb6c3c-068a-49cd-a98d-31ad151c6b79 2026-02-22T20:10:39.953 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:10:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:10:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:10:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:35 2026-02-22T20:10:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:39.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:39.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:10:39.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+error' 2026-02-22T20:10:39.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:10:39.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:39.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:10:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-22T20:10:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+error 2026-02-22T20:10:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern=split-brain 2026-02-22T20:10:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T20:10:43.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='split-brain: 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: effb6c3c-068a-49cd-a98d-31ad151c6b79 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+error 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: split-brain 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:40 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:10:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+error' 2026-02-22T20:10:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+error 2026-02-22T20:10:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*split-brain' 2026-02-22T20:10:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: split-brain 2026-02-22T20:10:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:10:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+error 2026-02-22T20:10:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:10:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+error 2026-02-22T20:10:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:10:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:10:43.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:10:43.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:10:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:40' 2026-02-22T20:10:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+error '2026-02-22 20:10:40' 2026-02-22T20:10:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+error 2026-02-22T20:10:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:10:40' 2026-02-22T20:10:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:10:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING160166028mirror.0trial09819.2.3-689-gf90f4eaetrueOK60206025mirror.1trial09819.2.3-689-gf90f4eaefalseOK60306037mirror.2trial09819.2.3-689-gf90f4eaefalseOK60346042mirror.3trial09819.2.3-689-gf90f4eaefalseOKsplit-braineffb6c3c-068a-49cd-a98d-31ad151c6b79up+errorsplit-brain60206025mirror.1trial0982026-02-22 20:10:40cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:10:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:10:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T20:10:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:40' 2026-02-22T20:10:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T20:10:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+error 2026-02-22T20:10:44.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+error 2026-02-22T20:10:44.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+error 2026-02-22T20:10:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+error 2026-02-22T20:10:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ request_resync_image cluster1 mirror split-brain image_id 2026-02-22T20:10:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image_id_var_name=image_id 2026-02-22T20:10:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_image_id cluster1 mirror split-brain 2026-02-22T20:10:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=split-brain 2026-02-22T20:10:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster=cluster1 info mirror/split-brain 2026-02-22T20:10:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T20:10:44.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'image_id='\''17895fa0e279'\''' 2026-02-22T20:10:44.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ image_id=17895fa0e279 2026-02-22T20:10:44.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval 'test -n "$image_id"' 2026-02-22T20:10:44.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -n 17895fa0e279 2026-02-22T20:10:44.069 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/split-brain 2026-02-22T20:10:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Flagged image for resync from primary 2026-02-22T20:10:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-02-22T20:10:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:10:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying 2026-02-22T20:10:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T20:10:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:45.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:45.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:45.106 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T20:10:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='split-brain: 2026-02-22T20:10:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: effb6c3c-068a-49cd-a98d-31ad151c6b79 2026-02-22T20:10:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+error 2026-02-22T20:10:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: split-brain 2026-02-22T20:10:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:10:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:40 2026-02-22T20:10:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:10:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:10:45.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:10:45.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:45.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:10:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T20:10:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:47.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T20:10:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd: error opening image split-brain: (2) No such file or directory 2026-02-22T20:10:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status= 2026-02-22T20:10:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:10:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 1 2026-02-22T20:10:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:10:51.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T20:10:51.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:51.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:51.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:51.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:51.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T20:10:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='split-brain: 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: effb6c3c-068a-49cd-a98d-31ad151c6b79 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791026,"replay_state":"idle"} 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:50 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: down+unknown 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: status not found 2026-02-22T20:10:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: ' 2026-02-22T20:10:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:10:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:10:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:10:51.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791026,"replay_state":"idle"} 2026-02-22T20:10:51.206 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:10:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:10:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:10:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:10:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:10:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:10:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:50' 2026-02-22T20:10:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-02-22 20:10:50' 2026-02-22T20:10:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:10:50' 2026-02-22T20:10:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:10:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:10:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:10:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING160166028mirror.0trial09819.2.3-689-gf90f4eaetrueOK60206025mirror.1trial09819.2.3-689-gf90f4eaefalseOK60306037mirror.2trial09819.2.3-689-gf90f4eaefalseOK60346042mirror.3trial09819.2.3-689-gf90f4eaefalseOKsplit-braineffb6c3c-068a-49cd-a98d-31ad151c6b79up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791026,"replay_state":"idle"}60206025mirror.1trial0982026-02-22 20:10:50cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559down+unknownstatus not found' 2026-02-22T20:10:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:10:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T20:10:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:50' 2026-02-22T20:10:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T20:10:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:10:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:10:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:10:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:10:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror split-brain 2026-02-22T20:10:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:10:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror split-brain 2026-02-22T20:10:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=split-brain 2026-02-22T20:10:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/split-brain 2026-02-22T20:10:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/split-brain 2026-02-22T20:10:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T20:10:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-22T20:10:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:10:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:10:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:10:52 TEST: check if removed 'images'\''' OMAP are removed 2026-02-22T20:10:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:10:52 TEST: check if removed images' OMAP are removed 2026-02-22T20:10:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster2 2026-02-22T20:10:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:10:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster2:0 2026-02-22T20:10:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:0 2026-02-22T20:10:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:10:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster2:0 2026-02-22T20:10:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster2 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster2 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster2:1 2026-02-22T20:10:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:1 2026-02-22T20:10:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:10:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster2:1 2026-02-22T20:10:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster2 2026-02-22T20:10:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T20:10:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T20:10:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster2 2026-02-22T20:10:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:10:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T20:10:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:10:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster2:2 2026-02-22T20:10:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:2 2026-02-22T20:10:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:10:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster2:2 2026-02-22T20:10:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster2 2026-02-22T20:10:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T20:10:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T20:10:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster2 2026-02-22T20:10:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:10:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T20:10:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:10:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster2:3 2026-02-22T20:10:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:3 2026-02-22T20:10:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:10:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T20:10:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster2:3 2026-02-22T20:10:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster2 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster2 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:10:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-22T20:10:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-22T20:10:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirroring 2026-02-22T20:10:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=status_global 2026-02-22T20:10:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T20:10:52.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T20:10:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-22T20:10:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T20:10:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=status_global 2026-02-22T20:10:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T20:10:52.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:52.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:52.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T20:10:52.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T20:10:52.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c status_global 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirroring 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=image_ 2026-02-22T20:10:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T20:10:52.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T20:10:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-22T20:10:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T20:10:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_ 2026-02-22T20:10:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T20:10:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T20:10:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T20:10:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_ 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T20:10:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=image_map 2026-02-22T20:10:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T20:10:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T20:10:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T20:10:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T20:10:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_map 2026-02-22T20:10:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T20:10:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T20:10:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T20:10:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_map 2026-02-22T20:10:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 1 = 0 2026-02-22T20:10:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=1 2026-02-22T20:10:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T20:10:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T20:10:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T20:10:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T20:10:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T20:10:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T20:10:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_map 2026-02-22T20:10:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T20:10:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_map 2026-02-22T20:10:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T20:10:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T20:10:53.488 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T20:10:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T20:10:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T20:10:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T20:10:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-22T20:10:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-22T20:10:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirroring 2026-02-22T20:10:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=status_global 2026-02-22T20:10:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T20:10:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:53.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=status_global 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T20:10:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c status_global 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirroring 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=image_ 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T20:10:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-22T20:10:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T20:10:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_ 2026-02-22T20:10:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T20:10:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_ 2026-02-22T20:10:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirroring 2026-02-22T20:10:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local filter=image_map 2026-02-22T20:10:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T20:10:53.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:10:53.535 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local filter=image_map 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -c image_map 2026-02-22T20:10:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T20:10:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 0 = 0 2026-02-22T20:10:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ error_code=0 2026-02-22T20:10:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set -e 2026-02-22T20:10:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T20:10:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T20:10:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: no blocklists' 2026-02-22T20:10:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:10:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:10:53.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:10:53 TEST: no blocklists 2026-02-22T20:10:53.558 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:10:53 TEST: no blocklists 2026-02-22T20:10:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS='--id admin' 2026-02-22T20:10:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster1 osd blocklist ls 2026-02-22T20:10:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -q 'listed 0 entries' 2026-02-22T20:10:53.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS='--id admin' 2026-02-22T20:10:53.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster2 osd blocklist ls 2026-02-22T20:10:53.761 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -q 'listed 0 entries' 2026-02-22T20:10:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:10:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T20:10:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: test partially synced demote snapshot sync after daemon restart' 2026-02-22T20:10:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:10:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:10:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:10:53 TEST: test partially synced demote snapshot sync after daemon restart 2026-02-22T20:10:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:10:53 TEST: test partially synced demote snapshot sync after daemon restart 2026-02-22T20:10:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image=test_demote_image 2026-02-22T20:10:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test_demote_image snapshot 10G 2026-02-22T20:10:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test_demote_image 10G 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:10:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 10G ']' 2026-02-22T20:10:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=10G 2026-02-22T20:10:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:10:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 10G --image-feature layering mirror/test_demote_image 2026-02-22T20:10:54.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:10:54.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:54.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:10:54.012 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_demote_image snapshot 2026-02-22T20:10:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:10:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_demote_image snapshot 2026-02-22T20:10:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:10:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_demote_image 2026-02-22T20:10:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_demote_image': 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 10 GiB in 2560 objects 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 184f214d6b2e 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.184f214d6b2e 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:10:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:10:54 2026 2026-02-22T20:10:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:10:54 2026 2026-02-22T20:10:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:10:54 2026 2026-02-22T20:10:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:10:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:10:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: 2d7bab14-eb9c-459e-afd7-2a1ab6c48432 2026-02-22T20:10:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:10:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-22T20:10:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:10:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:10:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:10:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:10:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:10:54.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:10:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:10:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.59 bytes/sec: 33 MiB/s 2026-02-22T20:10:54.373 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_demote_image 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T20:10:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:10:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:10:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:10:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T20:10:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:10:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:10:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:10:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:10:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:10:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:10:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:10:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:10:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:10:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:10:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:10:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:10:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:10:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:10:55.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:10:55.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:10:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:1 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:10:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:10:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:10:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:10:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:10:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:10:55.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:10:55.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:10:55.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:10:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-02-22T20:10:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:2 2026-02-22T20:10:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:10:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:10:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:2 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:10:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:2 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:10:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:10:55.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:10:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:10:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:10:55.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:10:55.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:10:55.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:10:55.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:10:55.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:55.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_demote_image 2026-02-22T20:10:55.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:3 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:3 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:3 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:10:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:10:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T20:10:55.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:10:55.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:10:55.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:10:55.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:10:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:10:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:10:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:10:55.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:55.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T20:10:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:10:56.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:10:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:10:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:10:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:10:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:10:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:10:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:10:56.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:10:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:10:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:10:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:10:56.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:10:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:10:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:10:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:10:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:10:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_demote_image 2026-02-22T20:10:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T20:10:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:10:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:10:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:10:57.734 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T20:10:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:10:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:10:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:10:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_demote_image 2026-02-22T20:10:57.735 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:10:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:10:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:10:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:10:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:10:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:10:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:10:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:10:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:10:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:10:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:10:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:10:57.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:10:57.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:10:57.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:57.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:57.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:10:57.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:10:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T20:10:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:10:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:10:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:10:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_demote_image 2026-02-22T20:10:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:10:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:10:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:10:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:10:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:10:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:10:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:10:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:57.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:57.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:10:57.819 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:10:57.874 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:10:57.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:10:57.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:10:57.875 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:10:57.876 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:10:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:10:57.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_demote_image 2026-02-22T20:10:57.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:57.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:57.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:57.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_demote_image 2026-02-22T20:10:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 192 2026-02-22T20:10:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T20:10:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:10:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T20:10:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-22T20:10:58.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:10:58.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:10:58.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-22T20:10:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-22T20:10:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:10:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:10:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:10:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:10:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:10:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:10:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:10:58.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:10:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=192 2026-02-22T20:10:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:10:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:10:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:10:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:10:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:10:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:10:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:10:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:10:58.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=191 2026-02-22T20:10:58.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 192 = 191 2026-02-22T20:10:58.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:10:58.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:10:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:10:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:10:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:10:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:10:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=192 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 192 = 192 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+replaying 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:10:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+replaying '' '' 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:11:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 2d7bab14-eb9c-459e-afd7-2a1ab6c48432 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791054,"replay_state":"idle"} 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.1 on trial098 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:56 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:57' 2026-02-22T20:11:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:11:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:11:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:11:00.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791054,"replay_state":"idle"} 2026-02-22T20:11:00.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:11:00.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:11:00.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:11:00.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:11:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:11:00.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.1 on trial098 2026-02-22T20:11:00.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:11:00.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:11:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:56' 2026-02-22T20:11:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+replaying '2026-02-22 20:10:56' 2026-02-22T20:11:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:11:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:10:56' 2026-02-22T20:11:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:11:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:11:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:11:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK160166028mirror.0trial09819.2.3-689-gf90f4eaetrueOK60206025mirror.1trial09819.2.3-689-gf90f4eaefalseOK60306037mirror.2trial09819.2.3-689-gf90f4eaefalseOK60346042mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_demote_image2d7bab14-eb9c-459e-afd7-2a1ab6c48432up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791054,"replay_state":"idle"}60206025mirror.1trial0982026-02-22 20:10:56cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:10:57' 2026-02-22T20:11:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:11:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:11:00.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:56' 2026-02-22T20:11:00.087 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:11:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:11:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:11:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:11:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+stopped 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:11:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:11:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+stopped '' '' 2026-02-22T20:11:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:11:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:11:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:11:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:11:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:11:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:11:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T20:11:01.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:11:01.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 2d7bab14-eb9c-459e-afd7-2a1ab6c48432 2026-02-22T20:11:01.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:57 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791054,"replay_state":"idle"} 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:10:56 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 191 .mirror.primary.2d7bab14-eb9c-459e-afd7-2a1ab6c48432.03ea87e9-4dad-4960-b67c-3a57f0d230a9 (peer_uuids:[]) 2026-02-22T20:11:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 192 .mirror.primary.2d7bab14-eb9c-459e-afd7-2a1ab6c48432.14966c9d-dc9c-4fb2-a26a-e972e7a45dec (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:11:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:11:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:11:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:11:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:11:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:11:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:11:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:11:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:11:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:11:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:11:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:11:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:11:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:57' 2026-02-22T20:11:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+stopped '2026-02-22 20:10:57' 2026-02-22T20:11:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:11:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:11:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:10:57' 2026-02-22T20:11:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:11:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:11:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:11:01.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK161706180mirror.0trial09819.2.3-689-gf90f4eaetrueOK61746186mirror.1trial09819.2.3-689-gf90f4eaefalseOK61816191mirror.2trial09819.2.3-689-gf90f4eaefalseOK61886194mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_demote_image2d7bab14-eb9c-459e-afd7-2a1ab6c48432up+stoppedlocal image is primary61706180mirror.0trial0982026-02-22 20:10:57cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791054,"replay_state":"idle"}2026-02-22 20:10:56' 2026-02-22T20:11:01.168 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:11:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:11:01.171 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:10:57' 2026-02-22T20:11:01.172 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:11:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:11:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:11:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:11:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:11:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:11:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:11:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_demote_image 2560 4194304 2026-02-22T20:11:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:11:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=2560 2026-02-22T20:11:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=4194304 2026-02-22T20:11:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 4194304 2026-02-22T20:11:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4194304 --io-threads 1 --io-total 10737418240 --io-pattern rand 2026-02-22T20:11:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4194304 io_threads 1 bytes 10737418240 pattern random 2026-02-22T20:11:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:11:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 1 634 640.141 2.5 GiB/s 2026-02-22T20:11:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 2 1239 622.509 2.4 GiB/s 2026-02-22T20:11:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 3 1827 610.981 2.4 GiB/s 2026-02-22T20:11:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 4 2403 602.223 2.4 GiB/s 2026-02-22T20:11:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 4 ops: 2560 ops/sec: 598.149 bytes/sec: 2.3 GiB/s 2026-02-22T20:11:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ demote_image cluster2 mirror test_demote_image 2026-02-22T20:11:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:11:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_demote_image 2026-02-22T20:11:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image demoted to non-primary 2026-02-22T20:11:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_complete_mirror_snapshot_id cluster2 mirror test_demote_image primary_snap_id 2026-02-22T20:11:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:11:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local -n _snap_id=primary_snap_id 2026-02-22T20:11:07.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:07.290 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '(//snapshots/snapshot[namespace/complete='\''true'\'']/id)[last()]' 2026-02-22T20:11:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ _snap_id=193 2026-02-22T20:11:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_non_primary_snap_present cluster1 mirror test_demote_image 193 2026-02-22T20:11:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snap_id=193 2026-02-22T20:11:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:11:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T20:11:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.1 2026-02-22T20:11:07.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:07.425 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot/namespace[primary_snap_id='\''193'\'']' 2026-02-22T20:11:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:11:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster1 -KILL 2026-02-22T20:11:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:11:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:11:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:11:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:0 -KILL 2026-02-22T20:11:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:11:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:11:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:11:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:11:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T20:11:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:0 2026-02-22T20:11:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:11:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:11:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:11:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:11:07.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:11:07.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:11:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=54165 2026-02-22T20:11:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 54165 ']' 2026-02-22T20:11:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill -KILL 54165 2026-02-22T20:11:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:11:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:11:07.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:11:07.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=54165 '$2 == pid {print; exit 1}' 2026-02-22T20:11:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:11:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:11:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=54165 '$2 == pid {print; exit 1}' 2026-02-22T20:11:07.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:11:07.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:11:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:11:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:11:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:11:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:11:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:11:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:11:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T20:11:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:11:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:11:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:11:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:11:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:11:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:11:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:0 2026-02-22T20:11:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:11:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:11:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:11:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:11:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:11:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:11:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:11:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:11:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:11:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:11:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:11:07.609 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:11:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:11:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:11:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:1 -KILL 2026-02-22T20:11:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:11:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:11:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:11:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:11:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T20:11:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:1 2026-02-22T20:11:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:11:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:11:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:11:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:11:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:11:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:11:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:11:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:11:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:11:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:11:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:11:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:11:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:11:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:11:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:11:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=54193 2026-02-22T20:11:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 54193 ']' 2026-02-22T20:11:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill -KILL 54193 2026-02-22T20:11:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:11:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:11:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:11:07.630 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=54193 '$2 == pid {print; exit 1}' 2026-02-22T20:11:07.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:11:07.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:11:07.657 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=54193 '$2 == pid {print; exit 1}' 2026-02-22T20:11:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:11:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:11:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:11:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:11:07.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:11:07.689 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:11:07.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:11:07.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:11:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T20:11:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:11:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:11:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:11:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:11:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:11:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:1 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:11:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:11:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:11:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:11:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:11:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:11:07.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:11:07.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:11:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:11:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:2 -KILL 2026-02-22T20:11:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:11:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:11:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:11:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:11:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:2 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:11:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:11:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:11:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:11:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:11:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:11:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:11:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=54243 2026-02-22T20:11:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 54243 ']' 2026-02-22T20:11:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill -KILL 54243 2026-02-22T20:11:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:11:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:11:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:11:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=54243 '$2 == pid {print; exit 1}' 2026-02-22T20:11:07.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:11:07.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:11:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=54243 '$2 == pid {print; exit 1}' 2026-02-22T20:11:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:11:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:11:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:11:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:11:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:11:07.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:11:07.809 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:11:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:11:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:11:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:11:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:11:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:11:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:11:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:11:07.821 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:11:07.822 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:2 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:11:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:11:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:11:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:11:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:11:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:11:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:11:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:3 -KILL 2026-02-22T20:11:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:11:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:11:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:11:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:11:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:3 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:11:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:11:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:11:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:11:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:11:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:11:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=54296 2026-02-22T20:11:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 54296 ']' 2026-02-22T20:11:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill -KILL 54296 2026-02-22T20:11:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:11:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:11:07.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:11:07.848 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=54296 '$2 == pid {print; exit 1}' 2026-02-22T20:11:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:11:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:11:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=54296 '$2 == pid {print; exit 1}' 2026-02-22T20:11:07.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:11:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:11:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:11:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:11:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:11:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:11:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:11:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:11:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T20:11:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:11:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:11:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:11:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:11:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:11:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:11:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:11:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:3 2026-02-22T20:11:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:11:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:11:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:11:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:11:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:11:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:11:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_snaps_json cluster1 mirror test_demote_image 2026-02-22T20:11:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:11:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:11:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_demote_image 2026-02-22T20:11:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 snap ls mirror/test_demote_image --all --format json 2026-02-22T20:11:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ SNAPS='[{"id":204,"name":".mirror.non_primary.2d7bab14-eb9c-459e-afd7-2a1ab6c48432.a157f6e9-5792-4d45-88f9-98d612779527","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 20:10:58 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":true,"primary_mirror_uuid":"7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559","primary_snap_id":192,"last_copied_object_number":2560}},{"id":205,"name":".mirror.non_primary.2d7bab14-eb9c-459e-afd7-2a1ab6c48432.5a425af3-bc18-4098-9646-442049629ff8","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 20:11:06 2026","namespace":{"type":"mirror","state":"demoted","mirror_peer_uuids":["df7543bc-b569-4771-a1ad-0eef6af1c1d2"],"complete":false,"primary_mirror_uuid":"7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559","primary_snap_id":193,"last_copied_object_number":680}}]' 2026-02-22T20:11:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "demoted" and .[-1].namespace["complete"] == false' 2026-02-22T20:11:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:mirrordemoteddf7543bc-b569-4771-a1ad-0eef6af1c1d2false7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559193538true 2026-02-22T20:11:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster1 2026-02-22T20:11:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:07.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:11:07.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:11:07.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:0 2026-02-22T20:11:07.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:11:07.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:11:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:11:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:11:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:11:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:1 2026-02-22T20:11:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:11:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:11:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:11:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:11:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:11:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:2 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:11:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T20:11:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:11:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:11:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:11:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:3 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T20:11:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:11:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:11:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T20:11:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:11:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:11:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:11:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T20:11:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:11:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:11:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T20:11:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-22T20:11:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:11:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:11:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-22T20:11:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-22T20:11:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:11:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:11:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:11:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.status 2026-02-22T20:11:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:11:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=193 2026-02-22T20:11:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:11:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:11:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.8 2026-02-22T20:11:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1.6 2026-02-22T20:11:11.222 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:11.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:11.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:11.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:11.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:11:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:11:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:15.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:15.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:11:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:19.363 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:19.364 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:11:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 8 2026-02-22T20:11:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:31.438 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:31.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 8 2026-02-22T20:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:39.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:39.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 16 2026-02-22T20:11:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:11:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:11:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:11:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:11:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:11:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:11:55.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:11:55.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:11:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:11:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:11:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:11:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:11:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = '' 2026-02-22T20:11:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:11:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 16 2026-02-22T20:12:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:12:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.status 2026-02-22T20:12:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T20:12:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:12:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=193 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 193 = 193 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+unknown 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-02-22T20:12:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:12:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T20:12:12.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:12:12.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 2d7bab14-eb9c-459e-afd7-2a1ab6c48432 2026-02-22T20:12:12.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:12:12.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:12:12.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:12:12.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:12 2026-02-22T20:12:12.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:12:12.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:12:12.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:12:12.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:12:12.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:08' 2026-02-22T20:12:12.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:12:12.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:12:12.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:12:12.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:12:12.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:12:12.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:12:12.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:12:12.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:12:12.588 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:12:12.589 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:12:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:12:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:12:12.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:12' 2026-02-22T20:12:12.592 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+unknown '2026-02-22 20:12:12' 2026-02-22T20:12:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:12:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:12:12' 2026-02-22T20:12:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:12:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING161706180mirror.0trial09819.2.3-689-gf90f4eaetrueOK61746186mirror.1trial09819.2.3-689-gf90f4eaefalseOK61816191mirror.2trial09819.2.3-689-gf90f4eaefalseOK61886194mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_demote_image2d7bab14-eb9c-459e-afd7-2a1ab6c48432up+unknownremote image is not primary61706180mirror.0trial0982026-02-22 20:12:12cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+unknownremote image is not primary2026-02-22 20:12:08' 2026-02-22T20:12:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:12:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:12:12.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:12' 2026-02-22T20:12:12.628 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:12:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:12:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:12:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:12:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:12:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+unknown 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-02-22T20:12:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:12:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T20:12:13.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:12:13.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 2d7bab14-eb9c-459e-afd7-2a1ab6c48432 2026-02-22T20:12:13.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:12:13.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:12:13.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:12:13.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:13 2026-02-22T20:12:13.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:12:13.669 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:12:13.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+unknown 2026-02-22T20:12:13.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: remote image is not primary 2026-02-22T20:12:13.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:12' 2026-02-22T20:12:13.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T20:12:13.670 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+unknown 2026-02-22T20:12:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:12:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: remote image is not primary 2026-02-22T20:12:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:12:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:12:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:12:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:12:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:12:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:12:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:12:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:12:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:13' 2026-02-22T20:12:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+unknown '2026-02-22 20:12:13' 2026-02-22T20:12:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+unknown 2026-02-22T20:12:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:12:13' 2026-02-22T20:12:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:12:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='WARNINGOKWARNING161836191mirror.0trial09819.2.3-689-gf90f4eaefalseOK61866195mirror.1trial09819.2.3-689-gf90f4eaefalseOK61926202mirror.2trial09819.2.3-689-gf90f4eaetrueOK61996205mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_demote_image2d7bab14-eb9c-459e-afd7-2a1ab6c48432up+unknownremote image is not primary61836191mirror.0trial0982026-02-22 20:12:13cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+unknownremote image is not primary2026-02-22 20:12:12' 2026-02-22T20:12:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:12:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:12:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:13' 2026-02-22T20:12:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:12:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+unknown 2026-02-22T20:12:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+unknown 2026-02-22T20:12:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+unknown 2026-02-22T20:12:13.726 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+unknown 2026-02-22T20:12:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ compare_images mirror test_demote_image 2026-02-22T20:12:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local ret=0 2026-02-22T20:12:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_demote_image.export 2026-02-22T20:12:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_demote_image.export 2026-02-22T20:12:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:12:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local rmt_export=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export 2026-02-22T20:12:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_demote_image.export 2026-02-22T20:12:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_demote_image.export 2026-02-22T20:12:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:12:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local loc_export=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:12:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:12:13.732 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 export mirror/test_demote_image /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export 2026-02-22T20:12:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T20:12:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 export mirror/test_demote_image /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:12:26.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T20:12:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cmp /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:12:30.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_demote_image.export /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_demote_image.export 2026-02-22T20:12:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test_demote_image 2026-02-22T20:12:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force= 2026-02-22T20:12:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_demote_image 2026-02-22T20:12:32.906 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:12:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster2 mirror test_demote_image 2026-02-22T20:12:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:33.912 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T20:12:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:12:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:12:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:12:33.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:12:33.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T20:12:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:12:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:12:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:12:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:12:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:33.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:12:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:12:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:12:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:12:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:12:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:12:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:12:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:12:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:12:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:12:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:12:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:12:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T20:12:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:12:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:12:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:12:35.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T20:12:35.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:12:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:12:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:12:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:12:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:12:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:12:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:12:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:12:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:12:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:12:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:12:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:12:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:12:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:12:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:12:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:12:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:12:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:12:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:12:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:12:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:12:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:12:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:12:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:12:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:12:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:12:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:12:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:12:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:12:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:12:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:12:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:12:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:12:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:12:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:12:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:12:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:12:36.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:36.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:36.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:36.007 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:12:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:12:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:12:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:12:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:12:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = stopped 2026-02-22T20:12:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 4 2026-02-22T20:12:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T20:12:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:12:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:12:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:12:40.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:12:40.077 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T20:12:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:12:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:12:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:12:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:12:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:12:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:12:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:12:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:12:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:12:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T20:12:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:12:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:12:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:12:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:12:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:12:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:12:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:12:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+stopped 2026-02-22T20:12:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:12:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+stopped '' '' 2026-02-22T20:12:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:12:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T20:12:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:12:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 2d7bab14-eb9c-459e-afd7-2a1ab6c48432 2026-02-22T20:12:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:38 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791152,"replay_state":"idle"} 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:37 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 206 .mirror.primary.2d7bab14-eb9c-459e-afd7-2a1ab6c48432.59a379ea-e5d2-4e80-9667-b9ead697038b (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:12:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:12:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:12:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:12:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:12:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:12:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:12:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:12:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:12:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:12:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:12:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:12:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:12:41.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:38' 2026-02-22T20:12:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+stopped '2026-02-22 20:12:38' 2026-02-22T20:12:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:12:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:12:38' 2026-02-22T20:12:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:12:41.309 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK161836191mirror.0trial09819.2.3-689-gf90f4eaefalseOK61866195mirror.1trial09819.2.3-689-gf90f4eaefalseOK61926202mirror.2trial09819.2.3-689-gf90f4eaetrueOK61996205mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_demote_image2d7bab14-eb9c-459e-afd7-2a1ab6c48432up+stoppedlocal image is primary61836191mirror.0trial0982026-02-22 20:12:38cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791152,"replay_state":"idle"}2026-02-22 20:12:37' 2026-02-22T20:12:41.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:12:41.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:12:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:38' 2026-02-22T20:12:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:12:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:12:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:12:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:12:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:12:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+replaying 2026-02-22T20:12:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:12:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:42.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+replaying '' '' 2026-02-22T20:12:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:12:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T20:12:42.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_demote_image: 2026-02-22T20:12:42.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: 2d7bab14-eb9c-459e-afd7-2a1ab6c48432 2026-02-22T20:12:42.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:12:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791152,"replay_state":"idle"} 2026-02-22T20:12:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:12:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:37 2026-02-22T20:12:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:12:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:12:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:12:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:12:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:38' 2026-02-22T20:12:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:12:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:12:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:12:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791152,"replay_state":"idle"} 2026-02-22T20:12:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:12:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:12:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:12:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:12:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:12:42.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:12:42.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:12:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:12:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:37' 2026-02-22T20:12:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+replaying '2026-02-22 20:12:37' 2026-02-22T20:12:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:12:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:12:37' 2026-02-22T20:12:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:12:42.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK161706180mirror.0trial09819.2.3-689-gf90f4eaetrueOK61746186mirror.1trial09819.2.3-689-gf90f4eaefalseOK61816191mirror.2trial09819.2.3-689-gf90f4eaefalseOK61886194mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_demote_image2d7bab14-eb9c-459e-afd7-2a1ab6c48432up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791152,"replay_state":"idle"}61706180mirror.0trial0982026-02-22 20:12:37cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:12:38' 2026-02-22T20:12:42.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:12:42.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T20:12:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:37' 2026-02-22T20:12:42.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T20:12:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:12:42.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:12:42.478 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:12:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:12:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster1 mirror test_demote_image 2026-02-22T20:12:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:12:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:12:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster1 mirror test_demote_image 2026-02-22T20:12:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_demote_image 2026-02-22T20:12:42.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_demote_image 2026-02-22T20:12:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 rm mirror/test_demote_image 2026-02-22T20:12:43.477 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T20:12:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ testlog 'TEST: force promote with a user snapshot' 2026-02-22T20:12:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ tee -a /tmp/tmp.5OeMBFOFTF/rbd-mirror.test.log 2026-02-22T20:12:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ date '+%F %T' 2026-02-22T20:12:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo 2026-02-22 20:12:43 TEST: force promote with a user snapshot 2026-02-22T20:12:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:2026-02-22 20:12:43 TEST: force promote with a user snapshot 2026-02-22T20:12:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ force_promote_image=test_force_promote_user 2026-02-22T20:12:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote_user snapshot 10G 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n snapshot ']' 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_image cluster2 mirror test_force_promote_user 10G 2026-02-22T20:12:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=128 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 10G ']' 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=10G 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ shift 2026-02-22T20:12:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 create --size 10G --image-feature layering mirror/test_force_promote_user 2026-02-22T20:12:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T20:12:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' image = image ']' 2026-02-22T20:12:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ enable_mirror cluster2 mirror test_force_promote_user snapshot 2026-02-22T20:12:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local mode=snapshot 2026-02-22T20:12:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_force_promote_user snapshot 2026-02-22T20:12:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Mirroring enabled 2026-02-22T20:12:44.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 info mirror/test_force_promote_user 2026-02-22T20:12:44.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_force_promote_user': 2026-02-22T20:12:44.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 10 GiB in 2560 objects 2026-02-22T20:12:44.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 1 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 1892f2b50cd0 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.1892f2b50cd0 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:12:43 2026 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:12:43 2026 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:12:43 2026 2026-02-22T20:12:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:12:44.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:12:44.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: f9bda3b4-002b-4f14-902e-c82a69c381c2 2026-02-22T20:12:44.508 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: true 2026-02-22T20:12:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-22T20:12:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:12:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:12:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:12:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:12:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:12:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:12:44.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:12:44.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 6250.17 bytes/sec: 24 MiB/s 2026-02-22T20:12:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:12:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T20:12:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:45.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:12:45.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:12:45.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:12:45.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:12:45.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:12:45.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:12:45.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:45.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:12:45.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:12:45.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:12:45.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:45.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:12:45.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:12:45.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:12:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:12:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:12:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:12:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:12:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:12:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:12:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:12:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:12:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:12:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:12:45.583 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:12:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:12:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:12:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:12:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:12:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote_user 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user started 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=started 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:45.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:46.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user started 2026-02-22T20:12:46.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:46.676 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=started 2026-02-22T20:12:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:12:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:12:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:12:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:12:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:12:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:12:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:12:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:12:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:12:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:12:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:12:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:12:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:12:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:12:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:12:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:12:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:12:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:12:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:12:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:12:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:12:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:12:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:12:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:12:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:12:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:12:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:12:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:12:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:12:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:12:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:12:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:12:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:12:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:12:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:12:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:12:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:12:46.747 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:12:46.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T20:12:46.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Replaying"' 2026-02-22T20:12:46.748 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:12:46.749 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: "state": "Replaying" 2026-02-22T20:12:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ current_state=started 2026-02-22T20:12:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test started = started 2026-02-22T20:12:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_force_promote_user 2026-02-22T20:12:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:12:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:12:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:12:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote_user 2026-02-22T20:12:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote_user 2026-02-22T20:12:47.461 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 196 2026-02-22T20:12:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_force_promote_user 2026-02-22T20:12:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_cluster=cluster1 2026-02-22T20:12:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster2-mirror-test_force_promote_user.status 2026-02-22T20:12:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster2-mirror-test_force_promote_user.status 2026-02-22T20:12:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:12:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_force_promote_user.status 2026-02-22T20:12:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ mkfname cluster1-mirror-test_force_promote_user.status 2026-02-22T20:12:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo cluster1-mirror-test_force_promote_user.status 2026-02-22T20:12:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T20:12:47.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local local_status_log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote_user.status 2026-02-22T20:12:47.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_force_promote_user /tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_force_promote_user.status 2026-02-22T20:12:47.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:47.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:47.472 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:47.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster2-mirror-test_force_promote_user.status 2026-02-22T20:12:47.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_force_promote_user --format xml 2026-02-22T20:12:47.473 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:12:47.501 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T20:12:47.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snapshot_id=196 2026-02-22T20:12:47.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:12:47.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:12:47.504 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.2 2026-02-22T20:12:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote_user.status 2026-02-22T20:12:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote_user.status 2026-02-22T20:12:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-02-22T20:12:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:12:47.737 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:12:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=195 2026-02-22T20:12:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 196 = 195 2026-02-22T20:12:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:12:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.4 2026-02-22T20:12:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote_user.status 2026-02-22T20:12:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote_user.status 2026-02-22T20:12:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-02-22T20:12:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:12:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ true 2026-02-22T20:12:48.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:12:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:-:1.1: Document is empty 2026-02-22T20:12:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 2026-02-22T20:12:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:^ 2026-02-22T20:12:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id= 2026-02-22T20:12:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 196 = '' 2026-02-22T20:12:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T20:12:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0.8 2026-02-22T20:12:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote_user.status 2026-02-22T20:12:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local log=/tmp/tmp.5OeMBFOFTF/cluster1-mirror-test_force_promote_user.status 2026-02-22T20:12:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-02-22T20:12:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T20:12:49.013 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T20:12:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local primary_snapshot_id=196 2026-02-22T20:12:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test 196 = 196 2026-02-22T20:12:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote_user 2026-02-22T20:12:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = journal ']' 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:50.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying '' '' 2026-02-22T20:12:50.018 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:12:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-02-22T20:12:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_force_promote_user: 2026-02-22T20:12:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: f9bda3b4-002b-4f14-902e-c82a69c381c2 2026-02-22T20:12:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:12:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791164,"replay_state":"idle"} 2026-02-22T20:12:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:12:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:45 2026-02-22T20:12:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:12:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:12:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:12:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:12:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:47' 2026-02-22T20:12:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T20:12:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+replaying 2026-02-22T20:12:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:12:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791164,"replay_state":"idle"} 2026-02-22T20:12:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:12:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:12:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:12:50.055 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:12:50.056 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:12:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:12:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:12:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:12:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:45' 2026-02-22T20:12:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+replaying '2026-02-22 20:12:45' 2026-02-22T20:12:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+replaying 2026-02-22T20:12:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:12:45' 2026-02-22T20:12:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:12:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK161836191mirror.0trial09819.2.3-689-gf90f4eaefalseOK61866195mirror.1trial09819.2.3-689-gf90f4eaefalseOK61926202mirror.2trial09819.2.3-689-gf90f4eaetrueOK61996205mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_force_promote_userf9bda3b4-002b-4f14-902e-c82a69c381c2up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791164,"replay_state":"idle"}61836191mirror.0trial0982026-02-22 20:12:45cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:12:47' 2026-02-22T20:12:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:12:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T20:12:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:45' 2026-02-22T20:12:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T20:12:50.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+replaying 2026-02-22T20:12:50.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+replaying 2026-02-22T20:12:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+replaying 2026-02-22T20:12:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+replaying 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:12:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-02-22T20:12:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:51.107 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:12:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:12:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:12:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-02-22T20:12:51.138 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_force_promote_user: 2026-02-22T20:12:51.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: f9bda3b4-002b-4f14-902e-c82a69c381c2 2026-02-22T20:12:51.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:12:51.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:12:51.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:12:51.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:47 2026-02-22T20:12:51.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:12:51.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:12:51.139 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+replaying 2026-02-22T20:12:51.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791164,"replay_state":"idle"} 2026-02-22T20:12:51.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:12:45 2026-02-22T20:12:51.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:12:51.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 195 .mirror.primary.f9bda3b4-002b-4f14-902e-c82a69c381c2.9449d034-e080-4e1b-b62d-0bde34968245 (peer_uuids:[]) 2026-02-22T20:12:51.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 196 .mirror.primary.f9bda3b4-002b-4f14-902e-c82a69c381c2.b3968dab-f1de-42b8-b4e6-688045ab4622 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:12:51.140 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:12:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:12:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:12:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:12:51.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:12:51.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:12:51.142 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:12:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:12:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:12:51.144 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:12:51.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:12:51.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:47' 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-02-22 20:12:47' 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:12:47' 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:12:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:12:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK161706180mirror.0trial09819.2.3-689-gf90f4eaetrueOK61746186mirror.1trial09819.2.3-689-gf90f4eaefalseOK61816191mirror.2trial09819.2.3-689-gf90f4eaefalseOK61886194mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_force_promote_userf9bda3b4-002b-4f14-902e-c82a69c381c2up+stoppedlocal image is primary61706180mirror.0trial0982026-02-22 20:12:47cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771791164,"replay_state":"idle"}2026-02-22 20:12:45' 2026-02-22T20:12:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:12:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T20:12:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:12:47' 2026-02-22T20:12:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T20:12:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:12:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:12:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:12:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:12:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-22T20:12:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:12:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:12:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:12:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:12:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:12:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:12:51.233 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:12:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.56 bytes/sec: 33 MiB/s 2026-02-22T20:12:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ create_snapshot cluster2 mirror test_force_promote_user snap1 2026-02-22T20:12:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:51.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:51.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap=snap1 2026-02-22T20:12:51.239 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 snap create mirror/test_force_promote_user@snap1 2026-02-22T20:12:52.079 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T20:12:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_force_promote_user 2560 4194304 2026-02-22T20:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=2560 2026-02-22T20:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size=4194304 2026-02-22T20:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n 4194304 2026-02-22T20:12:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4194304 --io-threads 1 --io-total 10737418240 --io-pattern rand 2026-02-22T20:12:52.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4194304 io_threads 1 bytes 10737418240 pattern random 2026-02-22T20:12:53.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:12:53.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 1 656 662.317 2.6 GiB/s 2026-02-22T20:12:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 2 1271 638.572 2.5 GiB/s 2026-02-22T20:12:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 3 1834 613.319 2.4 GiB/s 2026-02-22T20:12:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: 4 2366 592.952 2.3 GiB/s 2026-02-22T20:12:56.489 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 4 ops: 2560 ops/sec: 585.025 bytes/sec: 2.3 GiB/s 2026-02-22T20:12:56.494 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote_user 2026-02-22T20:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote_user 2026-02-22T20:12:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Snapshot ID: 198 2026-02-22T20:12:58.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_snap_present cluster1 mirror test_force_promote_user snap1 2026-02-22T20:12:58.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:12:58.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:12:58.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:12:58.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local snap_name=snap1 2026-02-22T20:12:58.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:12:58.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T20:12:58.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:12:59.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 info mirror/test_force_promote_user@snap1 2026-02-22T20:12:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:rbd image 'test_force_promote_user': 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: size 10 GiB in 2560 objects 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: order 22 (4 MiB objects) 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: snapshot_count: 3 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: id: 182fd0de889e 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: block_name_prefix: rbd_data.182fd0de889e 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: format: 2 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: features: layering, non-primary 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: op_features: 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: flags: 2026-02-22T20:12:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: create_timestamp: Sun Feb 22 20:12:45 2026 2026-02-22T20:12:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: access_timestamp: Sun Feb 22 20:12:45 2026 2026-02-22T20:12:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: modify_timestamp: Sun Feb 22 20:12:45 2026 2026-02-22T20:12:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: protected: False 2026-02-22T20:12:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring state: enabled 2026-02-22T20:12:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring mode: snapshot 2026-02-22T20:12:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring global id: f9bda3b4-002b-4f14-902e-c82a69c381c2 2026-02-22T20:12:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: mirroring primary: false 2026-02-22T20:12:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:12:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:13:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster1 -KILL 2026-02-22T20:13:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:13:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:13:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:13:00.326 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:0 -KILL 2026-02-22T20:13:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:13:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:13:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:13:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:13:00.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:0 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:13:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:13:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:13:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:13:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:13:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:13:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=57339 2026-02-22T20:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 57339 ']' 2026-02-22T20:13:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill -KILL 57339 2026-02-22T20:13:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:13:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:13:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:13:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=57339 '$2 == pid {print; exit 1}' 2026-02-22T20:13:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:13:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:13:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=57339 '$2 == pid {print; exit 1}' 2026-02-22T20:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:13:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:13:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:13:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:13:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:13:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T20:13:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:13:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:13:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:13:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:13:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:13:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:0 2026-02-22T20:13:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:13:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:13:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:13:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:13:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:1 -KILL 2026-02-22T20:13:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:13:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:13:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:13:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:13:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T20:13:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:1 2026-02-22T20:13:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:13:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:13:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:13:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:13:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:13:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:13:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:13:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:13:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:13:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:13:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=57367 2026-02-22T20:13:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 57367 ']' 2026-02-22T20:13:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill -KILL 57367 2026-02-22T20:13:00.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:13:00.468 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:13:00.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:13:00.469 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=57367 '$2 == pid {print; exit 1}' 2026-02-22T20:13:00.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:13:00.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:13:00.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=57367 '$2 == pid {print; exit 1}' 2026-02-22T20:13:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:13:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:13:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:13:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:13:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:13:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:13:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:13:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:13:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:13:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:13:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:13:00.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:00.557 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T20:13:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:13:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:13:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:13:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:00.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:13:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:1 2026-02-22T20:13:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:13:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:13:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:13:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:13:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:13:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:13:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:2 -KILL 2026-02-22T20:13:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:13:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:13:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:13:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:13:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T20:13:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:2 2026-02-22T20:13:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:13:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:13:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:13:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:13:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:13:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:13:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:13:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:13:00.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:13:00.595 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:13:00.596 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=57395 2026-02-22T20:13:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 57395 ']' 2026-02-22T20:13:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill -KILL 57395 2026-02-22T20:13:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:13:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:13:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:13:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=57395 '$2 == pid {print; exit 1}' 2026-02-22T20:13:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:13:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:13:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=57395 '$2 == pid {print; exit 1}' 2026-02-22T20:13:00.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:13:00.665 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:13:00.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:00.677 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:13:00.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:00.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:2 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:13:00.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:13:00.704 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:13:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:3 -KILL 2026-02-22T20:13:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:13:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig=-KILL 2026-02-22T20:13:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:13:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:3 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:13:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:13:00.717 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:13:00.718 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:13:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=57478 2026-02-22T20:13:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 57478 ']' 2026-02-22T20:13:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill -KILL 57478 2026-02-22T20:13:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:13:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:13:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:13:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=57478 '$2 == pid {print; exit 1}' 2026-02-22T20:13:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:13:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:13:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=57478 '$2 == pid {print; exit 1}' 2026-02-22T20:13:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:13:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:13:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:13:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:13:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:13:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:13:00.789 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:13:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:13:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:13:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:13:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:00.790 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:13:00.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:00.800 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:00.802 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:13:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:3 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:13:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:13:00.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T20:13:00.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ get_snaps_json cluster1 mirror test_force_promote_user 2026-02-22T20:13:00.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:13:00.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local pool=mirror 2026-02-22T20:13:00.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local image=test_force_promote_user 2026-02-22T20:13:00.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 snap ls mirror/test_force_promote_user --all --format json 2026-02-22T20:13:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ SNAPS='[{"id":208,"name":".mirror.non_primary.f9bda3b4-002b-4f14-902e-c82a69c381c2.a6a5c6c5-b0f7-4a66-aa1e-006562991479","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 20:12:47 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":true,"primary_mirror_uuid":"7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559","primary_snap_id":196,"last_copied_object_number":2560}},{"id":209,"name":"snap1","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 20:12:57 2026","namespace":{"type":"user"}},{"id":210,"name":".mirror.non_primary.f9bda3b4-002b-4f14-902e-c82a69c381c2.b0ef375d-806a-4ab7-9a22-dd56013c7b5a","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 20:12:58 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":false,"primary_mirror_uuid":"7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559","primary_snap_id":198,"last_copied_object_number":943}}]' 2026-02-22T20:13:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "non-primary" and .[-1].namespace["complete"] == false' 2026-02-22T20:13:00.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:true 2026-02-22T20:13:00.871 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ promote_image cluster1 mirror test_force_promote_user --force 2026-02-22T20:13:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:13:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:13:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:13:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local force=--force 2026-02-22T20:13:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote_user --force 2026-02-22T20:13:34.955 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:Image promoted to primary 2026-02-22T20:13:34.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirrors cluster1 2026-02-22T20:13:34.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:13:34.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:13:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:0 2026-02-22T20:13:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:13:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:0 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=0 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=0 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:13:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:13:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:1 2026-02-22T20:13:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:13:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:13:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:13:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:1 2026-02-22T20:13:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:13:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:13:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:13:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=1 2026-02-22T20:13:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:13:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=1 2026-02-22T20:13:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:13:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:13:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:2 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:2 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=2 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=2 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:13:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ start_mirror cluster1:3 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local val=cluster1:3 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster_var_name=cluster 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local instance_var_name=instance 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cluster=cluster1 2026-02-22T20:13:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ instance=3 2026-02-22T20:13:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval cluster=cluster1 2026-02-22T20:13:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ eval instance=3 2026-02-22T20:13:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:13:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:13:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:13:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote_user 2026-02-22T20:13:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:13:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:13:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:13:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-02-22T20:13:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:13:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:13:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:13:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:13:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:13:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:13:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:13:36.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-02-22T20:13:36.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:13:36.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:13:36.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:13:36.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:13:36.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:13:36.041 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster1 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster1 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:13:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:13:36.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:13:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:13:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:13:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:13:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:13:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:13:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:13:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:13:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:13:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:13:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:13:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:13:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:13:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:13:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:13:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:13:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:13:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:13:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:13:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:13:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:13:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:13:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:13:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 1 2026-02-22T20:13:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:13:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:13:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:13:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:13:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:13:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:37.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:37.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:37.382 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:37.385 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:13:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:13:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:13:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:13:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:13:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:13:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:13:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:13:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:13:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:13:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:13:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:13:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:13:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:13:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:13:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:13:37.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:13:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:13:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 2 2026-02-22T20:13:39.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:13:39.572 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:13:39.575 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:13:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:13:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:13:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:39.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:39.647 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:39.648 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:13:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:13:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:13:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:13:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:13:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:13:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:13:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:39.775 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:13:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:13:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:13:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:13:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:13:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 4 2026-02-22T20:13:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:13:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:13:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:13:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:13:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:43.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:43.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:43.910 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:13:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:13:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:13:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:13:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:43.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:43.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:43.975 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:43.979 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:13:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:13:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:13:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:13:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:13:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:44.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:44.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:44.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:44.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:13:44.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 8 2026-02-22T20:13:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:13:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:13:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:13:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:13:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:13:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:13:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:13:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:13:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:13:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:13:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:52.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:13:52.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:13:52.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:13:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:13:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:13:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:13:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:13:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:13:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:13:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:13:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:52.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:52.327 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:52.328 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:13:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:13:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:13:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:13:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:13:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:13:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:13:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:13:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:13:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:13:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:13:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:13:52.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:13:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:13:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:13:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:13:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:13:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:13:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:13:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:13:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:13:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:13:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:13:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:13:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:13:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:13:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:13:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:13:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:13:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:13:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:13:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:13:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:13:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:13:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 8 2026-02-22T20:14:00.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:14:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:14:00.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:14:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:14:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:14:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:14:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:14:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:00.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:14:00.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:14:00.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:14:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:14:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:14:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:14:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:14:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:14:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:14:00.490 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:14:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:14:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:14:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:14:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:14:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:14:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:14:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:14:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:14:00.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:00.561 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:00.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:00.562 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:14:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:14:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:14:00.621 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:14:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:14:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:14:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:14:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:14:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:14:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:14:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:00.690 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:14:00.690 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:14:00.690 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:14:00.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:14:00.691 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 8 2026-02-22T20:14:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:14:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:14:08.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:08.697 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:14:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:14:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:14:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:08.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:08.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:08.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:08.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:08.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:08.699 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:14:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:14:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:14:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:14:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:14:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:14:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:14:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:14:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:14:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:14:08.708 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:14:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:14:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:14:08.759 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:14:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:08.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:14:08.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:14:08.826 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:14:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:14:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:14:08.830 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:2 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:14:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:14:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:14:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:14:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:14:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:14:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:14:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:14:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:14:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:14:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:14:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:14:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:14:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:14:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:3 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:14:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:14:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:14:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:14:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:14:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:14:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T20:14:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T20:14:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:admin_socket: invalid command 2026-02-22T20:14:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:14:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 8 2026-02-22T20:14:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T20:14:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:14:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:14:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster1:0 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster1 2026-02-22T20:14:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster1 2026-02-22T20:14:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:14:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:14:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:14:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:14:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:14:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:14:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:14:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:14:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:14:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-02-22T20:14:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T20:14:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state=stopped 2026-02-22T20:14:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local s 2026-02-22T20:14:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:14:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:14:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T20:14:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local test_state=stopped 2026-02-22T20:14:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status_result 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local current_state=stopped 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ grep -i state 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_instance=cluster2 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance=cluster2 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local loop_instance 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:14:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sleep 0 2026-02-22T20:14:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T20:14:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ seq 0 3 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ shift 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local local_cluster=cluster2:0 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval local_cluster=cluster2 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ local_cluster=cluster2 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:14:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:14:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:14:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:14:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:14:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T20:14:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ return 0 2026-02-22T20:14:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T20:14:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo ' "state": "Stopped"' 2026-02-22T20:14:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep -i Replaying 2026-02-22T20:14:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test stopped = stopped 2026-02-22T20:14:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:14:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped 2026-02-22T20:14:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:14:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:14:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:14:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:14:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:14:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:14:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped '' '' 2026-02-22T20:14:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:14:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:14:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:14:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:14:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:14:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:14:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-02-22T20:14:19.161 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_force_promote_user: 2026-02-22T20:14:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: f9bda3b4-002b-4f14-902e-c82a69c381c2 2026-02-22T20:14:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:14:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:14:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:14:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:14:15 2026-02-22T20:14:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:14:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster2-DNE 2026-02-22T20:14:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:14:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:14:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:14:17 2026-02-22T20:14:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:14:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 212 .mirror.primary.f9bda3b4-002b-4f14-902e-c82a69c381c2.adca2d44-b073-411b-b4e8-3ec98c5c2f41 (peer_uuids:[df7543bc-b569-4771-a1ad-0eef6af1c1d2])' 2026-02-22T20:14:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:14:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:14:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:14:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:14:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:14:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:14:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:14:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:14:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:14:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:14:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:14:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:14:15' 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+stopped '2026-02-22 20:14:15' 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:14:15' 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:14:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T20:14:19.204 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK162986309mirror.0trial09819.2.3-689-gf90f4eaefalseOK63026315mirror.1trial09819.2.3-689-gf90f4eaefalseOK63066317mirror.2trial09819.2.3-689-gf90f4eaetrueOK63146320mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_force_promote_userf9bda3b4-002b-4f14-902e-c82a69c381c2up+stoppedlocal image is primary62986309mirror.0trial0982026-02-22 20:14:15cluster2-DNE7bc6647e-b49b-4fc7-ae1a-7c1ee7cb4559up+stoppedlocal image is primary2026-02-22 20:14:17' 2026-02-22T20:14:19.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:14:19.205 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T20:14:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:14:15' 2026-02-22T20:14:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T20:14:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:14:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:14:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:14:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:14:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:14:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:14:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-02-22T20:14:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:14:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:14:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:14:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T20:14:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:14:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-02-22T20:14:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local description_pattern= 2026-02-22T20:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local service_pattern= 2026-02-22T20:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:14:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-02-22T20:14:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='test_force_promote_user: 2026-02-22T20:14:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: global_id: f9bda3b4-002b-4f14-902e-c82a69c381c2 2026-02-22T20:14:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: service: mirror.0 on trial098 2026-02-22T20:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:14:17 2026-02-22T20:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: peer_sites: 2026-02-22T20:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: name: cluster1-DNE 2026-02-22T20:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: state: up+stopped 2026-02-22T20:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: description: local image is primary 2026-02-22T20:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: last_update: 2026-02-22 20:14:20 2026-02-22T20:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: snapshots: 2026-02-22T20:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 196 .mirror.primary.f9bda3b4-002b-4f14-902e-c82a69c381c2.b3968dab-f1de-42b8-b4e6-688045ab4622 (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d]) 2026-02-22T20:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: 198 .mirror.primary.f9bda3b4-002b-4f14-902e-c82a69c381c2.7b177d4a-d933-48db-8af2-911b81e5fb4d (peer_uuids:[e56cde10-e99e-4426-8926-9ba4aec5da6d])' 2026-02-22T20:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T20:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: state: up+stopped 2026-02-22T20:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^ description: .*' 2026-02-22T20:14:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: description: local image is primary 2026-02-22T20:14:20.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n '' ']' 2026-02-22T20:14:20.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:14:20.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep '^up+' 2026-02-22T20:14:20.251 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:14:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T20:14:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: service: mirror.0 on trial098 2026-02-22T20:14:20.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update 2026-02-22T20:14:20.253 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T20:14:20.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:14:17' 2026-02-22T20:14:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-02-22 20:14:17' 2026-02-22T20:14:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local state_pattern=up+stopped 2026-02-22T20:14:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local 'prev_last_update=2026-02-22 20:14:17' 2026-02-22T20:14:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local status 2026-02-22T20:14:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ CEPH_ARGS= 2026-02-22T20:14:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T20:14:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ status='OKOKOK161706180mirror.0trial09819.2.3-689-gf90f4eaetrueOK61746186mirror.1trial09819.2.3-689-gf90f4eaefalseOK61816191mirror.2trial09819.2.3-689-gf90f4eaefalseOK61886194mirror.3trial09819.2.3-689-gf90f4eaefalseOKtest_force_promote_userf9bda3b4-002b-4f14-902e-c82a69c381c2up+stoppedlocal image is primary61706180mirror.0trial0982026-02-22 20:14:17cluster1-DNEa81db8f5-c43a-4d9d-aa02-c74a333c3fadup+stoppedlocal image is primary2026-02-22 20:14:20' 2026-02-22T20:14:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local last_update state 2026-02-22T20:14:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T20:14:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ last_update='2026-02-22 20:14:17' 2026-02-22T20:14:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T20:14:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ state=up+stopped 2026-02-22T20:14:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo up+stopped 2026-02-22T20:14:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ grep up+stopped 2026-02-22T20:14:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:up+stopped 2026-02-22T20:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster1 mirror test_force_promote_user 100 2026-02-22T20:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:14:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:14:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:14:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:14:20.334 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:14:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:14:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 8333.55 bytes/sec: 33 MiB/s 2026-02-22T20:14:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-22T20:14:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local count=100 2026-02-22T20:14:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local size= 2026-02-22T20:14:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:20.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ size=4096 2026-02-22T20:14:20.353 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T20:14:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T20:14:20.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T20:14:20.412 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:elapsed: 0 ops: 100 ops/sec: 3125.08 bytes/sec: 12 MiB/s 2026-02-22T20:14:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster1 mirror test_force_promote_user 2026-02-22T20:14:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:14:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster1 mirror test_force_promote_user 2026-02-22T20:14:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:14:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote_user 2026-02-22T20:14:20.987 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T20:14:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote_user 2026-02-22T20:14:22.280 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T20:14:22.283 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:14:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image_retry cluster2 mirror test_force_promote_user 2026-02-22T20:14:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:22.284 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:22.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ remove_image cluster2 mirror test_force_promote_user 2026-02-22T20:14:22.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:22.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pool=mirror 2026-02-22T20:14:22.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local image=test_force_promote_user 2026-02-22T20:14:22.285 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote_user 2026-02-22T20:14:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T20:14:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote_user 2026-02-22T20:14:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T20:14:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ return 0 2026-02-22T20:14:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ cleanup 0 2026-02-22T20:14:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local error_code=0 2026-02-22T20:14:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ set +e 2026-02-22T20:14:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -ne 0 ']' 2026-02-22T20:14:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z '' ']' 2026-02-22T20:14:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster instance 2026-02-22T20:14:24.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T20:14:24.721 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T20:14:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:pool 'mirror' does not exist 2026-02-22T20:14:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T20:14:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T20:14:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:pool 'mirror' does not exist 2026-02-22T20:14:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T20:14:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T20:14:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:pool 'mirror_parent' does not exist 2026-02-22T20:14:27.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ CEPH_ARGS= 2026-02-22T20:14:27.045 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T20:14:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:pool 'mirror_parent' does not exist 2026-02-22T20:14:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T20:14:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster1 2026-02-22T20:14:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1 2026-02-22T20:14:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:14:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T20:14:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:0 2026-02-22T20:14:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:0 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:0 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:14:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:14:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:14:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:14:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=59253 2026-02-22T20:14:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 59253 ']' 2026-02-22T20:14:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 59253 2026-02-22T20:14:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59253 '$2 == pid {print; exit 1}' 2026-02-22T20:14:28.226 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:14:28.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:28.227 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59253 '$2 == pid {print; exit 1}' 2026-02-22T20:14:28.261 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:14:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:28.276 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:0 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:14:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:14:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T20:14:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T20:14:28.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:0 2026-02-22T20:14:28.291 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:0 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:0 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T20:14:28.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:14:28.302 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T20:14:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T20:14:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:1 2026-02-22T20:14:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:1 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:1 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:14:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:14:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:14:28.317 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:14:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=59281 2026-02-22T20:14:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 59281 ']' 2026-02-22T20:14:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 59281 2026-02-22T20:14:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59281 '$2 == pid {print; exit 1}' 2026-02-22T20:14:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:14:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59281 '$2 == pid {print; exit 1}' 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:14:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:14:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:14:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:14:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:14:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:14:28.400 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:28.401 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:1 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T20:14:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T20:14:28.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:1 2026-02-22T20:14:28.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:1 2026-02-22T20:14:28.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:28.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T20:14:28.417 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:1 2026-02-22T20:14:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:14:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T20:14:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:14:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T20:14:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:14:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T20:14:28.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:28.430 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T20:14:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:2 2026-02-22T20:14:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:14:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T20:14:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:2 2026-02-22T20:14:28.431 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:14:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:2 2026-02-22T20:14:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:14:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T20:14:28.432 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:14:28.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:14:28.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:14:28.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:14:28.433 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:14:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:14:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:14:28.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=59309 2026-02-22T20:14:28.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 59309 ']' 2026-02-22T20:14:28.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 59309 2026-02-22T20:14:28.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:28.441 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:28.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:28.442 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59309 '$2 == pid {print; exit 1}' 2026-02-22T20:14:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:14:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59309 '$2 == pid {print; exit 1}' 2026-02-22T20:14:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T20:14:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:2 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:14:28.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:14:28.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:14:28.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:14:28.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:14:28.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:28.527 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T20:14:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:28.538 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:2 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:2 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:2 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:28.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:14:28.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:28.541 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T20:14:28.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:14:28.552 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T20:14:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T20:14:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster1:3 2026-02-22T20:14:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster1:3 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster1:3 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T20:14:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster1 2026-02-22T20:14:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster1 2026-02-22T20:14:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:14:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:14:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:14:28.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:14:28.565 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:14:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=59360 2026-02-22T20:14:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 59360 ']' 2026-02-22T20:14:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 59360 2026-02-22T20:14:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59360 '$2 == pid {print; exit 1}' 2026-02-22T20:14:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 59360 0.2 0.0 504664 35080 ? Ssl 20:13 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:14:28.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:28.602 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 1 2026-02-22T20:14:29.603 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:29.604 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59360 '$2 == pid {print; exit 1}' 2026-02-22T20:14:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:ubuntu 59360 0.2 0.0 504664 35080 ? Ssl 20:13 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T20:14:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 2 2026-02-22T20:14:31.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:31.640 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59360 '$2 == pid {print; exit 1}' 2026-02-22T20:14:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:14:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=59360 '$2 == pid {print; exit 1}' 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:14:31.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:14:31.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:14:31.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:14:31.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:14:31.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:31.714 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:14:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T20:14:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster1:3 2026-02-22T20:14:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:14:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T20:14:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:14:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:14:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:14:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster1 2026-02-22T20:14:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster1 2026-02-22T20:14:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:14:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T20:14:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T20:14:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster1:3 2026-02-22T20:14:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1:3 2026-02-22T20:14:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T20:14:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster1:3 2026-02-22T20:14:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster1 2026-02-22T20:14:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:14:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T20:14:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster1 2026-02-22T20:14:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster1 2026-02-22T20:14:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:14:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T20:14:31.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:14:31.752 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T20:14:31.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T20:14:31.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirrors cluster2 2026-02-22T20:14:31.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2 2026-02-22T20:14:31.753 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:31.754 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ seq 0 3 2026-02-22T20:14:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T20:14:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:0 2026-02-22T20:14:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:14:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster2:0 2026-02-22T20:14:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2:0 2026-02-22T20:14:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:14:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:0 2026-02-22T20:14:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:14:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:31.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T20:14:31.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster2 2026-02-22T20:14:31.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster2 2026-02-22T20:14:31.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=0 2026-02-22T20:14:31.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=0 2026-02-22T20:14:31.757 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T20:14:31.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T20:14:31.768 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T20:14:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=55404 2026-02-22T20:14:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 55404 ']' 2026-02-22T20:14:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 55404 2026-02-22T20:14:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:31.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:31.771 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=55404 '$2 == pid {print; exit 1}' 2026-02-22T20:14:31.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:14:31.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:31.804 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=55404 '$2 == pid {print; exit 1}' 2026-02-22T20:14:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-22T20:14:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:0 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:14:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:14:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:14:31.850 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:14:31.851 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T20:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:0 2026-02-22T20:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T20:14:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:14:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T20:14:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster2:0 2026-02-22T20:14:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:0 2026-02-22T20:14:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T20:14:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:0 2026-02-22T20:14:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=0 2026-02-22T20:14:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T20:14:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:14:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=0 2026-02-22T20:14:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=0 2026-02-22T20:14:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T20:14:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T20:14:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T20:14:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster2:1 '' 2026-02-22T20:14:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:1 2026-02-22T20:14:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:14:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster2:1 2026-02-22T20:14:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2:1 2026-02-22T20:14:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:14:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:1 2026-02-22T20:14:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:14:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T20:14:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster2 2026-02-22T20:14:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster2 2026-02-22T20:14:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=1 2026-02-22T20:14:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=1 2026-02-22T20:14:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T20:14:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T20:14:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T20:14:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=55432 2026-02-22T20:14:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 55432 ']' 2026-02-22T20:14:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 55432 2026-02-22T20:14:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=55432 '$2 == pid {print; exit 1}' 2026-02-22T20:14:31.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:14:31.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:31.927 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=55432 '$2 == pid {print; exit 1}' 2026-02-22T20:14:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-02-22T20:14:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:1 2026-02-22T20:14:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:14:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:14:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:14:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:14:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T20:14:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:1 2026-02-22T20:14:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:14:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T20:14:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:14:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:14:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:14:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:14:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:14:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:14:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T20:14:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:14:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T20:14:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster2:1 2026-02-22T20:14:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:1 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:1 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=1 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=1 2026-02-22T20:14:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=1 2026-02-22T20:14:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T20:14:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T20:14:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T20:14:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster2:2 '' 2026-02-22T20:14:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:2 2026-02-22T20:14:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:14:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster2:2 2026-02-22T20:14:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2:2 2026-02-22T20:14:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:14:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:2 2026-02-22T20:14:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:14:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T20:14:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster2 2026-02-22T20:14:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster2 2026-02-22T20:14:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=2 2026-02-22T20:14:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=2 2026-02-22T20:14:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T20:14:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T20:14:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T20:14:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=55461 2026-02-22T20:14:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 55461 ']' 2026-02-22T20:14:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 55461 2026-02-22T20:14:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=55461 '$2 == pid {print; exit 1}' 2026-02-22T20:14:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:14:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=55461 '$2 == pid {print; exit 1}' 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:2 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:14:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:14:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T20:14:32.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:14:32.116 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster2:2 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:2 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:2 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=2 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=2 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=2 2026-02-22T20:14:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T20:14:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T20:14:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T20:14:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T20:14:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ stop_mirror cluster2:3 '' 2026-02-22T20:14:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local cluster=cluster2:3 2026-02-22T20:14:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local sig= 2026-02-22T20:14:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test -n '' 2026-02-22T20:14:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ local pid 2026-02-22T20:14:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ daemon_pid_file cluster2:3 2026-02-22T20:14:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster=cluster2:3 2026-02-22T20:14:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local val=cluster2:3 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local cluster_var_name=cluster 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local instance_var_name=instance 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval cluster=cluster2 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ cluster=cluster2 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ eval instance=3 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ instance=3 2026-02-22T20:14:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T20:14:32.144 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T20:14:32.145 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T20:14:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ pid=55536 2026-02-22T20:14:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -n 55536 ']' 2026-02-22T20:14:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ kill 55536 2026-02-22T20:14:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T20:14:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ sleep 0 2026-02-22T20:14:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=55536 '$2 == pid {print; exit 1}' 2026-02-22T20:14:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ break 2026-02-22T20:14:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ ps auxww 2026-02-22T20:14:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ awk -v pid=55536 '$2 == pid {print; exit 1}' 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:3 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster1 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T20:14:32.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:14:32.228 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:14:32.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T20:14:32.230 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local local_cluster=cluster2:3 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=local_cluster 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval local_cluster=cluster2 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ local_cluster=cluster2 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:32.231 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T20:14:32.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:14:32.242 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ daemon_pid_file cluster2:3 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster=cluster2:3 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local val=cluster2:3 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local cluster_var_name=cluster 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ local instance_var_name=instance 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ cluster=cluster2 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ instance=3 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval cluster=cluster2 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ cluster=cluster2 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ eval instance=3 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ instance=3 2026-02-22T20:14:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T20:14:32.254 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T20:14:32.255 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T20:14:32.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' -z 1 ']' 2026-02-22T20:14:32.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ test '' = /tmp/tmp.5OeMBFOFTF 2026-02-22T20:14:32.256 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ rm -Rf /tmp/tmp.5OeMBFOFTF 2026-02-22T20:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T20:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ echo OK 2026-02-22T20:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stdout:OK 2026-02-22T20:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial098.stderr:+ exit 0 2026-02-22T20:14:32.260 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-22T20:14:32.260 DEBUG:teuthology.orchestra.run.trial098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-22T20:14:32.315 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror.sh'] on cluster1.client.mirror... 2026-02-22T20:14:32.316 DEBUG:teuthology.orchestra.run.trial098:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-22T20:14:32.659 DEBUG:teuthology.parallel:result is None 2026-02-22T20:14:32.659 DEBUG:teuthology.orchestra.run.trial098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T20:14:32.668 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T20:14:32.668 DEBUG:teuthology.orchestra.run.trial098:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T20:14:32.715 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T20:14:32.715 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T20:14:32.731 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T20:14:32.731 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T20:14:32.905 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T20:14:32.906 INFO:teuthology.orchestra.run.trial098.stderr:dumped all 2026-02-22T20:14:32.917 INFO:teuthology.orchestra.run.trial098.stdout:{"pg_ready":true,"pg_map":{"version":1213,"stamp":"2026-02-22T20:14:31.032607+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":232,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":327588,"kb_used_data":12204,"kb_used_omap":4,"kb_used_meta":315323,"kb_avail":282787932,"statfs":{"total":289910292480,"available":289574842368,"internally_reserved":0,"allocated":12496896,"data_stored":7815344,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":322891090},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7265042654,"num_objects":-1953,"num_object_clones":-168,"num_object_copies":-3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1953,"num_whiteouts":0,"num_read":-95434,"num_read_kb":-16253751,"num_write":-17630,"num_write_kb":-27728418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.929515"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1001,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.129537+0000","last_change":"2026-02-22T19:50:54.861563+0000","last_active":"2026-02-22T20:14:26.129537+0000","last_peered":"2026-02-22T20:14:26.129537+0000","last_clean":"2026-02-22T20:14:26.129537+0000","last_became_active":"2026-02-22T19:50:52.861375+0000","last_became_peered":"2026-02-22T19:50:52.861375+0000","last_unstale":"2026-02-22T20:14:26.129537+0000","last_undegraded":"2026-02-22T20:14:26.129537+0000","last_fullsized":"2026-02-22T20:14:26.129537+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:48.919040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030579899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1001,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.131077+0000","last_change":"2026-02-22T19:50:54.861720+0000","last_active":"2026-02-22T20:14:26.131077+0000","last_peered":"2026-02-22T20:14:26.131077+0000","last_clean":"2026-02-22T20:14:26.131077+0000","last_became_active":"2026-02-22T19:50:52.861559+0000","last_became_peered":"2026-02-22T19:50:52.861559+0000","last_unstale":"2026-02-22T20:14:26.131077+0000","last_undegraded":"2026-02-22T20:14:26.131077+0000","last_fullsized":"2026-02-22T20:14:26.131077+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:54.548580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042885800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1002,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.132183+0000","last_change":"2026-02-22T19:50:54.861777+0000","last_active":"2026-02-22T20:14:26.132183+0000","last_peered":"2026-02-22T20:14:26.132183+0000","last_clean":"2026-02-22T20:14:26.132183+0000","last_became_active":"2026-02-22T19:50:52.861577+0000","last_became_peered":"2026-02-22T19:50:52.861577+0000","last_unstale":"2026-02-22T20:14:26.132183+0000","last_undegraded":"2026-02-22T20:14:26.132183+0000","last_fullsized":"2026-02-22T20:14:26.132183+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:41:03.314267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046371299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1002,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.128253+0000","last_change":"2026-02-22T19:50:54.861943+0000","last_active":"2026-02-22T20:14:26.128253+0000","last_peered":"2026-02-22T20:14:26.128253+0000","last_clean":"2026-02-22T20:14:26.128253+0000","last_became_active":"2026-02-22T19:50:52.860973+0000","last_became_peered":"2026-02-22T19:50:52.860973+0000","last_unstale":"2026-02-22T20:14:26.128253+0000","last_undegraded":"2026-02-22T20:14:26.128253+0000","last_fullsized":"2026-02-22T20:14:26.128253+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:16.079406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063366900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1003,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.634336+0000","last_change":"2026-02-22T19:50:54.863357+0000","last_active":"2026-02-22T20:14:26.634336+0000","last_peered":"2026-02-22T20:14:26.634336+0000","last_clean":"2026-02-22T20:14:26.634336+0000","last_became_active":"2026-02-22T19:50:52.861077+0000","last_became_peered":"2026-02-22T19:50:52.861077+0000","last_unstale":"2026-02-22T20:14:26.634336+0000","last_undegraded":"2026-02-22T20:14:26.634336+0000","last_fullsized":"2026-02-22T20:14:26.634336+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:21:08.931023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064508999999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1002,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.119598+0000","last_change":"2026-02-22T19:50:55.134568+0000","last_active":"2026-02-22T20:14:26.119598+0000","last_peered":"2026-02-22T20:14:26.119598+0000","last_clean":"2026-02-22T20:14:26.119598+0000","last_became_active":"2026-02-22T19:50:52.860722+0000","last_became_peered":"2026-02-22T19:50:52.860722+0000","last_unstale":"2026-02-22T20:14:26.119598+0000","last_undegraded":"2026-02-22T20:14:26.119598+0000","last_fullsized":"2026-02-22T20:14:26.119598+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:06:27.453930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026535399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1002,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.119721+0000","last_change":"2026-02-22T19:50:55.134562+0000","last_active":"2026-02-22T20:14:26.119721+0000","last_peered":"2026-02-22T20:14:26.119721+0000","last_clean":"2026-02-22T20:14:26.119721+0000","last_became_active":"2026-02-22T19:50:52.860764+0000","last_became_peered":"2026-02-22T19:50:52.860764+0000","last_unstale":"2026-02-22T20:14:26.119721+0000","last_undegraded":"2026-02-22T20:14:26.119721+0000","last_fullsized":"2026-02-22T20:14:26.119721+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:34:00.906689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028186500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1003,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.130579+0000","last_change":"2026-02-22T19:50:54.861491+0000","last_active":"2026-02-22T20:14:26.130579+0000","last_peered":"2026-02-22T20:14:26.130579+0000","last_clean":"2026-02-22T20:14:26.130579+0000","last_became_active":"2026-02-22T19:50:52.860289+0000","last_became_peered":"2026-02-22T19:50:52.860289+0000","last_unstale":"2026-02-22T20:14:26.130579+0000","last_undegraded":"2026-02-22T20:14:26.130579+0000","last_fullsized":"2026-02-22T20:14:26.130579+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:13:55.531886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027309799999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1092,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.130206+0000","last_change":"2026-02-22T19:50:48.847243+0000","last_active":"2026-02-22T20:14:26.130206+0000","last_peered":"2026-02-22T20:14:26.130206+0000","last_clean":"2026-02-22T20:14:26.130206+0000","last_became_active":"2026-02-22T19:50:48.846636+0000","last_became_peered":"2026-02-22T19:50:48.846636+0000","last_unstale":"2026-02-22T20:14:26.130206+0000","last_undegraded":"2026-02-22T20:14:26.130206+0000","last_fullsized":"2026-02-22T20:14:26.130206+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_clean_scrub_stamp":"2026-02-22T19:50:47.840459+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:42:30.969871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738656,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":85140,"kb_used_data":4032,"kb_used_omap":1,"kb_used_meta":81086,"kb_avail":94286700,"statfs":{"total":96636764160,"available":96549580800,"internally_reserved":0,"allocated":4128768,"data_stored":2212900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":83032518},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738657,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121232,"kb_used_data":4092,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250608,"statfs":{"total":96636764160,"available":96512622592,"internally_reserved":0,"allocated":4190208,"data_stored":2803287,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738655,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121216,"kb_used_data":4080,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250624,"statfs":{"total":96636764160,"available":96512638976,"internally_reserved":0,"allocated":4177920,"data_stored":2799157,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:14:32.919 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T20:14:33.063 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T20:14:33.064 INFO:teuthology.orchestra.run.trial098.stderr:dumped all 2026-02-22T20:14:33.072 INFO:teuthology.orchestra.run.trial098.stdout:{"pg_ready":true,"pg_map":{"version":1214,"stamp":"2026-02-22T20:14:33.032797+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":327628,"kb_used_data":12240,"kb_used_omap":4,"kb_used_meta":315323,"kb_avail":282787892,"statfs":{"total":289910292480,"available":289574801408,"internally_reserved":0,"allocated":12533760,"data_stored":7830385,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":322891090},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7265042654,"num_objects":-1953,"num_object_clones":-168,"num_object_copies":-3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1953,"num_whiteouts":0,"num_read":-95434,"num_read_kb":-16253751,"num_write":-17630,"num_write_kb":-27728418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.000968"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1003,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.123393+0000","last_change":"2026-02-22T19:50:54.861563+0000","last_active":"2026-02-22T20:14:28.123393+0000","last_peered":"2026-02-22T20:14:28.123393+0000","last_clean":"2026-02-22T20:14:28.123393+0000","last_became_active":"2026-02-22T19:50:52.861375+0000","last_became_peered":"2026-02-22T19:50:52.861375+0000","last_unstale":"2026-02-22T20:14:28.123393+0000","last_undegraded":"2026-02-22T20:14:28.123393+0000","last_fullsized":"2026-02-22T20:14:28.123393+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:48.919040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030579899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1003,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.125594+0000","last_change":"2026-02-22T19:50:54.861720+0000","last_active":"2026-02-22T20:14:28.125594+0000","last_peered":"2026-02-22T20:14:28.125594+0000","last_clean":"2026-02-22T20:14:28.125594+0000","last_became_active":"2026-02-22T19:50:52.861559+0000","last_became_peered":"2026-02-22T19:50:52.861559+0000","last_unstale":"2026-02-22T20:14:28.125594+0000","last_undegraded":"2026-02-22T20:14:28.125594+0000","last_fullsized":"2026-02-22T20:14:28.125594+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:54.548580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042885800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.125140+0000","last_change":"2026-02-22T19:50:54.861777+0000","last_active":"2026-02-22T20:14:28.125140+0000","last_peered":"2026-02-22T20:14:28.125140+0000","last_clean":"2026-02-22T20:14:28.125140+0000","last_became_active":"2026-02-22T19:50:52.861577+0000","last_became_peered":"2026-02-22T19:50:52.861577+0000","last_unstale":"2026-02-22T20:14:28.125140+0000","last_undegraded":"2026-02-22T20:14:28.125140+0000","last_fullsized":"2026-02-22T20:14:28.125140+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:41:03.314267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046371299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.122627+0000","last_change":"2026-02-22T19:50:54.861943+0000","last_active":"2026-02-22T20:14:28.122627+0000","last_peered":"2026-02-22T20:14:28.122627+0000","last_clean":"2026-02-22T20:14:28.122627+0000","last_became_active":"2026-02-22T19:50:52.860973+0000","last_became_peered":"2026-02-22T19:50:52.860973+0000","last_unstale":"2026-02-22T20:14:28.122627+0000","last_undegraded":"2026-02-22T20:14:28.122627+0000","last_fullsized":"2026-02-22T20:14:28.122627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:16.079406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063366900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1005,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:29.211075+0000","last_change":"2026-02-22T19:50:54.863357+0000","last_active":"2026-02-22T20:14:29.211075+0000","last_peered":"2026-02-22T20:14:29.211075+0000","last_clean":"2026-02-22T20:14:29.211075+0000","last_became_active":"2026-02-22T19:50:52.861077+0000","last_became_peered":"2026-02-22T19:50:52.861077+0000","last_unstale":"2026-02-22T20:14:29.211075+0000","last_undegraded":"2026-02-22T20:14:29.211075+0000","last_fullsized":"2026-02-22T20:14:29.211075+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:21:08.931023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064508999999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.481452+0000","last_change":"2026-02-22T19:50:55.134568+0000","last_active":"2026-02-22T20:14:28.481452+0000","last_peered":"2026-02-22T20:14:28.481452+0000","last_clean":"2026-02-22T20:14:28.481452+0000","last_became_active":"2026-02-22T19:50:52.860722+0000","last_became_peered":"2026-02-22T19:50:52.860722+0000","last_unstale":"2026-02-22T20:14:28.481452+0000","last_undegraded":"2026-02-22T20:14:28.481452+0000","last_fullsized":"2026-02-22T20:14:28.481452+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:06:27.453930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026535399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.482120+0000","last_change":"2026-02-22T19:50:55.134562+0000","last_active":"2026-02-22T20:14:28.482120+0000","last_peered":"2026-02-22T20:14:28.482120+0000","last_clean":"2026-02-22T20:14:28.482120+0000","last_became_active":"2026-02-22T19:50:52.860764+0000","last_became_peered":"2026-02-22T19:50:52.860764+0000","last_unstale":"2026-02-22T20:14:28.482120+0000","last_undegraded":"2026-02-22T20:14:28.482120+0000","last_fullsized":"2026-02-22T20:14:28.482120+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:34:00.906689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028186500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1005,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.124812+0000","last_change":"2026-02-22T19:50:54.861491+0000","last_active":"2026-02-22T20:14:28.124812+0000","last_peered":"2026-02-22T20:14:28.124812+0000","last_clean":"2026-02-22T20:14:28.124812+0000","last_became_active":"2026-02-22T19:50:52.860289+0000","last_became_peered":"2026-02-22T19:50:52.860289+0000","last_unstale":"2026-02-22T20:14:28.124812+0000","last_undegraded":"2026-02-22T20:14:28.124812+0000","last_fullsized":"2026-02-22T20:14:28.124812+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:13:55.531886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027309799999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1094,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.125033+0000","last_change":"2026-02-22T19:50:48.847243+0000","last_active":"2026-02-22T20:14:28.125033+0000","last_peered":"2026-02-22T20:14:28.125033+0000","last_clean":"2026-02-22T20:14:28.125033+0000","last_became_active":"2026-02-22T19:50:48.846636+0000","last_became_peered":"2026-02-22T19:50:48.846636+0000","last_unstale":"2026-02-22T20:14:28.125033+0000","last_undegraded":"2026-02-22T20:14:28.125033+0000","last_fullsized":"2026-02-22T20:14:28.125033+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_clean_scrub_stamp":"2026-02-22T19:50:47.840459+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:42:30.969871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738657,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":85148,"kb_used_data":4040,"kb_used_omap":1,"kb_used_meta":81086,"kb_avail":94286692,"statfs":{"total":96636764160,"available":96549572608,"internally_reserved":0,"allocated":4136960,"data_stored":2216537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":83032518},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738658,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121240,"kb_used_data":4100,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250600,"statfs":{"total":96636764160,"available":96512614400,"internally_reserved":0,"allocated":4198400,"data_stored":2806924,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738656,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121240,"kb_used_data":4100,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250600,"statfs":{"total":96636764160,"available":96512614400,"internally_reserved":0,"allocated":4198400,"data_stored":2806924,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:14:33.073 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T20:14:33.073 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T20:14:33.225 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T20:14:33.226 INFO:teuthology.orchestra.run.trial098.stderr:dumped all 2026-02-22T20:14:33.235 INFO:teuthology.orchestra.run.trial098.stdout:{"pg_ready":true,"pg_map":{"version":1214,"stamp":"2026-02-22T20:14:33.032797+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":327628,"kb_used_data":12240,"kb_used_omap":4,"kb_used_meta":315323,"kb_avail":282787892,"statfs":{"total":289910292480,"available":289574801408,"internally_reserved":0,"allocated":12533760,"data_stored":7830385,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":322891090},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7265042654,"num_objects":-1953,"num_object_clones":-168,"num_object_copies":-3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1953,"num_whiteouts":0,"num_read":-95434,"num_read_kb":-16253751,"num_write":-17630,"num_write_kb":-27728418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.000968"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1003,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.123393+0000","last_change":"2026-02-22T19:50:54.861563+0000","last_active":"2026-02-22T20:14:28.123393+0000","last_peered":"2026-02-22T20:14:28.123393+0000","last_clean":"2026-02-22T20:14:28.123393+0000","last_became_active":"2026-02-22T19:50:52.861375+0000","last_became_peered":"2026-02-22T19:50:52.861375+0000","last_unstale":"2026-02-22T20:14:28.123393+0000","last_undegraded":"2026-02-22T20:14:28.123393+0000","last_fullsized":"2026-02-22T20:14:28.123393+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:48.919040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030579899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1003,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.125594+0000","last_change":"2026-02-22T19:50:54.861720+0000","last_active":"2026-02-22T20:14:28.125594+0000","last_peered":"2026-02-22T20:14:28.125594+0000","last_clean":"2026-02-22T20:14:28.125594+0000","last_became_active":"2026-02-22T19:50:52.861559+0000","last_became_peered":"2026-02-22T19:50:52.861559+0000","last_unstale":"2026-02-22T20:14:28.125594+0000","last_undegraded":"2026-02-22T20:14:28.125594+0000","last_fullsized":"2026-02-22T20:14:28.125594+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:54.548580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042885800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.125140+0000","last_change":"2026-02-22T19:50:54.861777+0000","last_active":"2026-02-22T20:14:28.125140+0000","last_peered":"2026-02-22T20:14:28.125140+0000","last_clean":"2026-02-22T20:14:28.125140+0000","last_became_active":"2026-02-22T19:50:52.861577+0000","last_became_peered":"2026-02-22T19:50:52.861577+0000","last_unstale":"2026-02-22T20:14:28.125140+0000","last_undegraded":"2026-02-22T20:14:28.125140+0000","last_fullsized":"2026-02-22T20:14:28.125140+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:41:03.314267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046371299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.122627+0000","last_change":"2026-02-22T19:50:54.861943+0000","last_active":"2026-02-22T20:14:28.122627+0000","last_peered":"2026-02-22T20:14:28.122627+0000","last_clean":"2026-02-22T20:14:28.122627+0000","last_became_active":"2026-02-22T19:50:52.860973+0000","last_became_peered":"2026-02-22T19:50:52.860973+0000","last_unstale":"2026-02-22T20:14:28.122627+0000","last_undegraded":"2026-02-22T20:14:28.122627+0000","last_fullsized":"2026-02-22T20:14:28.122627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:16.079406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063366900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1005,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:29.211075+0000","last_change":"2026-02-22T19:50:54.863357+0000","last_active":"2026-02-22T20:14:29.211075+0000","last_peered":"2026-02-22T20:14:29.211075+0000","last_clean":"2026-02-22T20:14:29.211075+0000","last_became_active":"2026-02-22T19:50:52.861077+0000","last_became_peered":"2026-02-22T19:50:52.861077+0000","last_unstale":"2026-02-22T20:14:29.211075+0000","last_undegraded":"2026-02-22T20:14:29.211075+0000","last_fullsized":"2026-02-22T20:14:29.211075+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:21:08.931023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064508999999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.481452+0000","last_change":"2026-02-22T19:50:55.134568+0000","last_active":"2026-02-22T20:14:28.481452+0000","last_peered":"2026-02-22T20:14:28.481452+0000","last_clean":"2026-02-22T20:14:28.481452+0000","last_became_active":"2026-02-22T19:50:52.860722+0000","last_became_peered":"2026-02-22T19:50:52.860722+0000","last_unstale":"2026-02-22T20:14:28.481452+0000","last_undegraded":"2026-02-22T20:14:28.481452+0000","last_fullsized":"2026-02-22T20:14:28.481452+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:06:27.453930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026535399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.482120+0000","last_change":"2026-02-22T19:50:55.134562+0000","last_active":"2026-02-22T20:14:28.482120+0000","last_peered":"2026-02-22T20:14:28.482120+0000","last_clean":"2026-02-22T20:14:28.482120+0000","last_became_active":"2026-02-22T19:50:52.860764+0000","last_became_peered":"2026-02-22T19:50:52.860764+0000","last_unstale":"2026-02-22T20:14:28.482120+0000","last_undegraded":"2026-02-22T20:14:28.482120+0000","last_fullsized":"2026-02-22T20:14:28.482120+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:34:00.906689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028186500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1005,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.124812+0000","last_change":"2026-02-22T19:50:54.861491+0000","last_active":"2026-02-22T20:14:28.124812+0000","last_peered":"2026-02-22T20:14:28.124812+0000","last_clean":"2026-02-22T20:14:28.124812+0000","last_became_active":"2026-02-22T19:50:52.860289+0000","last_became_peered":"2026-02-22T19:50:52.860289+0000","last_unstale":"2026-02-22T20:14:28.124812+0000","last_undegraded":"2026-02-22T20:14:28.124812+0000","last_fullsized":"2026-02-22T20:14:28.124812+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:13:55.531886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027309799999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1094,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.125033+0000","last_change":"2026-02-22T19:50:48.847243+0000","last_active":"2026-02-22T20:14:28.125033+0000","last_peered":"2026-02-22T20:14:28.125033+0000","last_clean":"2026-02-22T20:14:28.125033+0000","last_became_active":"2026-02-22T19:50:48.846636+0000","last_became_peered":"2026-02-22T19:50:48.846636+0000","last_unstale":"2026-02-22T20:14:28.125033+0000","last_undegraded":"2026-02-22T20:14:28.125033+0000","last_fullsized":"2026-02-22T20:14:28.125033+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_clean_scrub_stamp":"2026-02-22T19:50:47.840459+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:42:30.969871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738657,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":85148,"kb_used_data":4040,"kb_used_omap":1,"kb_used_meta":81086,"kb_avail":94286692,"statfs":{"total":96636764160,"available":96549572608,"internally_reserved":0,"allocated":4136960,"data_stored":2216537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":83032518},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738658,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121240,"kb_used_data":4100,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250600,"statfs":{"total":96636764160,"available":96512614400,"internally_reserved":0,"allocated":4198400,"data_stored":2806924,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738656,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121240,"kb_used_data":4100,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250600,"statfs":{"total":96636764160,"available":96512614400,"internally_reserved":0,"allocated":4198400,"data_stored":2806924,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:14:33.236 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T20:14:33.387 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T20:14:33.388 INFO:teuthology.orchestra.run.trial098.stdout:{"epoch":503,"fsid":"c78b27d3-80f9-4252-ae60-d838c71871f8","created":"2026-02-22T19:50:37.162442+0000","modified":"2026-02-22T20:14:28.115852+0000","last_up_change":"2026-02-22T19:50:45.832310+0000","last_in_change":"2026-02-22T19:50:37.678649+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T19:50:46.859457+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T19:50:51.265384+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0b31b72f-873e-4231-92d2-2bf473c7fb49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6800","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6801","nonce":1925958386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6802","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6804","nonce":1925958386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6810","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6812","nonce":1925958386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6806","nonce":1925958386},{"type":"v1","addr":"10.20.193.98:6808","nonce":1925958386}]},"public_addr":"10.20.193.98:6801/1925958386","cluster_addr":"10.20.193.98:6804/1925958386","heartbeat_back_addr":"10.20.193.98:6812/1925958386","heartbeat_front_addr":"10.20.193.98:6808/1925958386","state":["exists","up"]},{"osd":1,"uuid":"12f18cab-1f27-4a1f-9342-68723365376d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6816","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6817","nonce":2796279274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6818","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6819","nonce":2796279274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6822","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6823","nonce":2796279274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6820","nonce":2796279274},{"type":"v1","addr":"10.20.193.98:6821","nonce":2796279274}]},"public_addr":"10.20.193.98:6817/2796279274","cluster_addr":"10.20.193.98:6819/2796279274","heartbeat_back_addr":"10.20.193.98:6823/2796279274","heartbeat_front_addr":"10.20.193.98:6821/2796279274","state":["exists","up"]},{"osd":2,"uuid":"96497b35-d35f-466d-a113-5b0925d9f0c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6803","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6805","nonce":2602009039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6807","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6809","nonce":2602009039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6814","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6815","nonce":2602009039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.98:6811","nonce":2602009039},{"type":"v1","addr":"10.20.193.98:6813","nonce":2602009039}]},"public_addr":"10.20.193.98:6805/2602009039","cluster_addr":"10.20.193.98:6809/2602009039","heartbeat_back_addr":"10.20.193.98:6815/2602009039","heartbeat_front_addr":"10.20.193.98:6813/2602009039","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.399936+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.672521+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:44.246703+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":196,"length":3}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T20:14:34.399 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-22T20:14:34.400 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T20:14:34.482 INFO:teuthology.orchestra.run.trial098.stdout:{ 2026-02-22T20:14:34.483 INFO:teuthology.orchestra.run.trial098.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T20:14:34.483 INFO:teuthology.orchestra.run.trial098.stdout:} 2026-02-22T20:14:34.491 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-22T20:14:34.637 INFO:teuthology.orchestra.run.trial098.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T20:14:34.648 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-22T20:14:34.648 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T20:14:34.729 INFO:teuthology.orchestra.run.trial098.stdout:{ 2026-02-22T20:14:34.729 INFO:teuthology.orchestra.run.trial098.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T20:14:34.729 INFO:teuthology.orchestra.run.trial098.stdout:} 2026-02-22T20:14:34.742 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-22T20:14:34.893 INFO:teuthology.orchestra.run.trial098.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T20:14:34.904 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-22T20:14:34.904 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T20:14:34.979 INFO:teuthology.orchestra.run.trial098.stdout:{ 2026-02-22T20:14:34.979 INFO:teuthology.orchestra.run.trial098.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T20:14:34.980 INFO:teuthology.orchestra.run.trial098.stdout:} 2026-02-22T20:14:34.989 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-22T20:14:35.138 INFO:teuthology.orchestra.run.trial098.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T20:14:35.150 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T20:14:35.302 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T20:14:35.302 INFO:teuthology.orchestra.run.trial098.stderr:dumped all 2026-02-22T20:14:35.312 INFO:teuthology.orchestra.run.trial098.stdout:{"pg_ready":true,"pg_map":{"version":1215,"stamp":"2026-02-22T20:14:35.033027+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":327628,"kb_used_data":12240,"kb_used_omap":4,"kb_used_meta":315323,"kb_avail":282787892,"statfs":{"total":289910292480,"available":289574801408,"internally_reserved":0,"allocated":12533760,"data_stored":7830385,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":322891090},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-19,"num_objects":-18,"num_object_clones":0,"num_object_copies":-36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-18,"num_whiteouts":0,"num_read":-9669,"num_read_kb":-12181,"num_write":-885,"num_write_kb":-1912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.921851"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1003,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.123393+0000","last_change":"2026-02-22T19:50:54.861563+0000","last_active":"2026-02-22T20:14:28.123393+0000","last_peered":"2026-02-22T20:14:28.123393+0000","last_clean":"2026-02-22T20:14:28.123393+0000","last_became_active":"2026-02-22T19:50:52.861375+0000","last_became_peered":"2026-02-22T19:50:52.861375+0000","last_unstale":"2026-02-22T20:14:28.123393+0000","last_undegraded":"2026-02-22T20:14:28.123393+0000","last_fullsized":"2026-02-22T20:14:28.123393+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:48.919040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030579899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1003,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.125594+0000","last_change":"2026-02-22T19:50:54.861720+0000","last_active":"2026-02-22T20:14:28.125594+0000","last_peered":"2026-02-22T20:14:28.125594+0000","last_clean":"2026-02-22T20:14:28.125594+0000","last_became_active":"2026-02-22T19:50:52.861559+0000","last_became_peered":"2026-02-22T19:50:52.861559+0000","last_unstale":"2026-02-22T20:14:28.125594+0000","last_undegraded":"2026-02-22T20:14:28.125594+0000","last_fullsized":"2026-02-22T20:14:28.125594+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:54.548580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042885800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.125140+0000","last_change":"2026-02-22T19:50:54.861777+0000","last_active":"2026-02-22T20:14:28.125140+0000","last_peered":"2026-02-22T20:14:28.125140+0000","last_clean":"2026-02-22T20:14:28.125140+0000","last_became_active":"2026-02-22T19:50:52.861577+0000","last_became_peered":"2026-02-22T19:50:52.861577+0000","last_unstale":"2026-02-22T20:14:28.125140+0000","last_undegraded":"2026-02-22T20:14:28.125140+0000","last_fullsized":"2026-02-22T20:14:28.125140+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:41:03.314267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046371299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.122627+0000","last_change":"2026-02-22T19:50:54.861943+0000","last_active":"2026-02-22T20:14:28.122627+0000","last_peered":"2026-02-22T20:14:28.122627+0000","last_clean":"2026-02-22T20:14:28.122627+0000","last_became_active":"2026-02-22T19:50:52.860973+0000","last_became_peered":"2026-02-22T19:50:52.860973+0000","last_unstale":"2026-02-22T20:14:28.122627+0000","last_undegraded":"2026-02-22T20:14:28.122627+0000","last_fullsized":"2026-02-22T20:14:28.122627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:16.079406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063366900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1005,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:29.211075+0000","last_change":"2026-02-22T19:50:54.863357+0000","last_active":"2026-02-22T20:14:29.211075+0000","last_peered":"2026-02-22T20:14:29.211075+0000","last_clean":"2026-02-22T20:14:29.211075+0000","last_became_active":"2026-02-22T19:50:52.861077+0000","last_became_peered":"2026-02-22T19:50:52.861077+0000","last_unstale":"2026-02-22T20:14:29.211075+0000","last_undegraded":"2026-02-22T20:14:29.211075+0000","last_fullsized":"2026-02-22T20:14:29.211075+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:21:08.931023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064508999999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.481452+0000","last_change":"2026-02-22T19:50:55.134568+0000","last_active":"2026-02-22T20:14:28.481452+0000","last_peered":"2026-02-22T20:14:28.481452+0000","last_clean":"2026-02-22T20:14:28.481452+0000","last_became_active":"2026-02-22T19:50:52.860722+0000","last_became_peered":"2026-02-22T19:50:52.860722+0000","last_unstale":"2026-02-22T20:14:28.481452+0000","last_undegraded":"2026-02-22T20:14:28.481452+0000","last_fullsized":"2026-02-22T20:14:28.481452+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:06:27.453930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026535399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1004,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.482120+0000","last_change":"2026-02-22T19:50:55.134562+0000","last_active":"2026-02-22T20:14:28.482120+0000","last_peered":"2026-02-22T20:14:28.482120+0000","last_clean":"2026-02-22T20:14:28.482120+0000","last_became_active":"2026-02-22T19:50:52.860764+0000","last_became_peered":"2026-02-22T19:50:52.860764+0000","last_unstale":"2026-02-22T20:14:28.482120+0000","last_undegraded":"2026-02-22T20:14:28.482120+0000","last_fullsized":"2026-02-22T20:14:28.482120+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:34:00.906689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028186500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1005,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.124812+0000","last_change":"2026-02-22T19:50:54.861491+0000","last_active":"2026-02-22T20:14:28.124812+0000","last_peered":"2026-02-22T20:14:28.124812+0000","last_clean":"2026-02-22T20:14:28.124812+0000","last_became_active":"2026-02-22T19:50:52.860289+0000","last_became_peered":"2026-02-22T19:50:52.860289+0000","last_unstale":"2026-02-22T20:14:28.124812+0000","last_undegraded":"2026-02-22T20:14:28.124812+0000","last_fullsized":"2026-02-22T20:14:28.124812+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:51.850871+0000","last_clean_scrub_stamp":"2026-02-22T19:50:51.850871+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:13:55.531886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027309799999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1094,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.125033+0000","last_change":"2026-02-22T19:50:48.847243+0000","last_active":"2026-02-22T20:14:28.125033+0000","last_peered":"2026-02-22T20:14:28.125033+0000","last_clean":"2026-02-22T20:14:28.125033+0000","last_became_active":"2026-02-22T19:50:48.846636+0000","last_became_peered":"2026-02-22T19:50:48.846636+0000","last_unstale":"2026-02-22T20:14:28.125033+0000","last_undegraded":"2026-02-22T20:14:28.125033+0000","last_fullsized":"2026-02-22T20:14:28.125033+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:47.840459+0000","last_clean_scrub_stamp":"2026-02-22T19:50:47.840459+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:42:30.969871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738657,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":85148,"kb_used_data":4040,"kb_used_omap":1,"kb_used_meta":81086,"kb_avail":94286692,"statfs":{"total":96636764160,"available":96549572608,"internally_reserved":0,"allocated":4136960,"data_stored":2216537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":83032518},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738658,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121240,"kb_used_data":4100,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250600,"statfs":{"total":96636764160,"available":96512614400,"internally_reserved":0,"allocated":4198400,"data_stored":2806924,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738656,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121240,"kb_used_data":4100,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250600,"statfs":{"total":96636764160,"available":96512614400,"internally_reserved":0,"allocated":4198400,"data_stored":2806924,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:14:35.318 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T19:50:51.850871+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=51, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=14, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:14:35.319 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T19:50:51.850871+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=51, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=14, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:14:35.319 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T19:50:51.850871+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=51, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=14, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:14:35.319 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T19:50:51.850871+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=51, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=14, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:14:35.319 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T19:50:51.850871+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=51, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=14, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:14:35.320 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T19:50:51.850871+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=51, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=14, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:14:35.320 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T19:50:51.850871+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=51, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=14, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:14:35.320 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T19:50:51.850871+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=51, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=14, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:14:35.320 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T19:50:47.840459+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=47, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=14, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:14:35.320 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T20:14:55.321 DEBUG:teuthology.orchestra.run.trial098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T20:14:55.472 INFO:teuthology.orchestra.run.trial098.stdout: 2026-02-22T20:14:55.472 INFO:teuthology.orchestra.run.trial098.stderr:dumped all 2026-02-22T20:14:55.482 INFO:teuthology.orchestra.run.trial098.stdout:{"pg_ready":true,"pg_map":{"version":1225,"stamp":"2026-02-22T20:14:55.034714+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":327616,"kb_used_data":12240,"kb_used_omap":4,"kb_used_meta":315323,"kb_avail":282787904,"statfs":{"total":289910292480,"available":289574813696,"internally_reserved":0,"allocated":12533760,"data_stored":7830385,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":322891090},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001021"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1013,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:38.238310+0000","last_change":"2026-02-22T20:14:38.238310+0000","last_active":"2026-02-22T20:14:38.238310+0000","last_peered":"2026-02-22T20:14:38.238310+0000","last_clean":"2026-02-22T20:14:38.238310+0000","last_became_active":"2026-02-22T19:50:52.861375+0000","last_became_peered":"2026-02-22T19:50:52.861375+0000","last_unstale":"2026-02-22T20:14:38.238310+0000","last_undegraded":"2026-02-22T20:14:38.238310+0000","last_fullsized":"2026-02-22T20:14:38.238310+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:14:38.238216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:14:38.238216+0000","last_clean_scrub_stamp":"2026-02-22T20:14:38.238216+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:02:57.006802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030579899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1013,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:39.235897+0000","last_change":"2026-02-22T20:14:39.235897+0000","last_active":"2026-02-22T20:14:39.235897+0000","last_peered":"2026-02-22T20:14:39.235897+0000","last_clean":"2026-02-22T20:14:39.235897+0000","last_became_active":"2026-02-22T19:50:52.861559+0000","last_became_peered":"2026-02-22T19:50:52.861559+0000","last_unstale":"2026-02-22T20:14:39.235897+0000","last_undegraded":"2026-02-22T20:14:39.235897+0000","last_fullsized":"2026-02-22T20:14:39.235897+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:14:39.235817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:14:39.235817+0000","last_clean_scrub_stamp":"2026-02-22T20:14:39.235817+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T20:35:01.046511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042885800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1013,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:40.263461+0000","last_change":"2026-02-22T20:14:40.263479+0000","last_active":"2026-02-22T20:14:40.263461+0000","last_peered":"2026-02-22T20:14:40.263461+0000","last_clean":"2026-02-22T20:14:40.263461+0000","last_became_active":"2026-02-22T19:50:52.861577+0000","last_became_peered":"2026-02-22T19:50:52.861577+0000","last_unstale":"2026-02-22T20:14:40.263461+0000","last_undegraded":"2026-02-22T20:14:40.263461+0000","last_fullsized":"2026-02-22T20:14:40.263461+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:14:40.263400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:14:40.263400+0000","last_clean_scrub_stamp":"2026-02-22T20:14:40.263400+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:07:58.677436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046371299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1013,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:37.192988+0000","last_change":"2026-02-22T20:14:37.193017+0000","last_active":"2026-02-22T20:14:37.192988+0000","last_peered":"2026-02-22T20:14:37.192988+0000","last_clean":"2026-02-22T20:14:37.192988+0000","last_became_active":"2026-02-22T19:50:52.860973+0000","last_became_peered":"2026-02-22T19:50:52.860973+0000","last_unstale":"2026-02-22T20:14:37.192988+0000","last_undegraded":"2026-02-22T20:14:37.192988+0000","last_fullsized":"2026-02-22T20:14:37.192988+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:14:37.192932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:14:37.192932+0000","last_clean_scrub_stamp":"2026-02-22T20:14:37.192932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:36:08.704048+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00063366900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"503'3","reported_seq":1015,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:35.188346+0000","last_change":"2026-02-22T20:14:35.188368+0000","last_active":"2026-02-22T20:14:35.188346+0000","last_peered":"2026-02-22T20:14:35.188346+0000","last_clean":"2026-02-22T20:14:35.188346+0000","last_became_active":"2026-02-22T19:50:52.861077+0000","last_became_peered":"2026-02-22T19:50:52.861077+0000","last_unstale":"2026-02-22T20:14:35.188346+0000","last_undegraded":"2026-02-22T20:14:35.188346+0000","last_fullsized":"2026-02-22T20:14:35.188346+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"503'3","last_scrub_stamp":"2026-02-22T20:14:35.188285+0000","last_deep_scrub":"503'3","last_deep_scrub_stamp":"2026-02-22T20:14:35.188285+0000","last_clean_scrub_stamp":"2026-02-22T20:14:35.188285+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:17:06.880123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064508999999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1013,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:36.185185+0000","last_change":"2026-02-22T20:14:36.185208+0000","last_active":"2026-02-22T20:14:36.185185+0000","last_peered":"2026-02-22T20:14:36.185185+0000","last_clean":"2026-02-22T20:14:36.185185+0000","last_became_active":"2026-02-22T19:50:52.860722+0000","last_became_peered":"2026-02-22T19:50:52.860722+0000","last_unstale":"2026-02-22T20:14:36.185185+0000","last_undegraded":"2026-02-22T20:14:36.185185+0000","last_fullsized":"2026-02-22T20:14:36.185185+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:14:36.185129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:14:36.185129+0000","last_clean_scrub_stamp":"2026-02-22T20:14:36.185129+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:34:57.605141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026535399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1013,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:35.174524+0000","last_change":"2026-02-22T20:14:35.174547+0000","last_active":"2026-02-22T20:14:35.174524+0000","last_peered":"2026-02-22T20:14:35.174524+0000","last_clean":"2026-02-22T20:14:35.174524+0000","last_became_active":"2026-02-22T19:50:52.860764+0000","last_became_peered":"2026-02-22T19:50:52.860764+0000","last_unstale":"2026-02-22T20:14:35.174524+0000","last_undegraded":"2026-02-22T20:14:35.174524+0000","last_fullsized":"2026-02-22T20:14:35.174524+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:14:35.174423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:14:35.174423+0000","last_clean_scrub_stamp":"2026-02-22T20:14:35.174423+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:06:31.631104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028186500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"503'2","reported_seq":1015,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:36.174821+0000","last_change":"2026-02-22T20:14:36.174877+0000","last_active":"2026-02-22T20:14:36.174821+0000","last_peered":"2026-02-22T20:14:36.174821+0000","last_clean":"2026-02-22T20:14:36.174821+0000","last_became_active":"2026-02-22T19:50:52.860289+0000","last_became_peered":"2026-02-22T19:50:52.860289+0000","last_unstale":"2026-02-22T20:14:36.174821+0000","last_undegraded":"2026-02-22T20:14:36.174821+0000","last_fullsized":"2026-02-22T20:14:36.174821+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"503'2","last_scrub_stamp":"2026-02-22T20:14:36.174719+0000","last_deep_scrub":"503'2","last_deep_scrub_stamp":"2026-02-22T20:14:36.174719+0000","last_clean_scrub_stamp":"2026-02-22T20:14:36.174719+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:22:25.756902+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00027309799999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"503'74","reported_seq":1105,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-22T20:14:35.187726+0000","last_change":"2026-02-22T20:14:35.187756+0000","last_active":"2026-02-22T20:14:35.187726+0000","last_peered":"2026-02-22T20:14:35.187726+0000","last_clean":"2026-02-22T20:14:35.187726+0000","last_became_active":"2026-02-22T19:50:48.846636+0000","last_became_peered":"2026-02-22T19:50:48.846636+0000","last_unstale":"2026-02-22T20:14:35.187726+0000","last_undegraded":"2026-02-22T20:14:35.187726+0000","last_fullsized":"2026-02-22T20:14:35.187726+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"503'74","last_scrub_stamp":"2026-02-22T20:14:35.187667+0000","last_deep_scrub":"503'74","last_deep_scrub_stamp":"2026-02-22T20:14:35.187667+0000","last_clean_scrub_stamp":"2026-02-22T20:14:35.187667+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:21:35.181906+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11722,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738661,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":85144,"kb_used_data":4040,"kb_used_omap":1,"kb_used_meta":81086,"kb_avail":94286696,"statfs":{"total":96636764160,"available":96549576704,"internally_reserved":0,"allocated":4136960,"data_stored":2216537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":83032518},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738662,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121236,"kb_used_data":4100,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250604,"statfs":{"total":96636764160,"available":96512618496,"internally_reserved":0,"allocated":4198400,"data_stored":2806924,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738660,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":121236,"kb_used_data":4100,"kb_used_omap":1,"kb_used_meta":117118,"kb_avail":94250604,"statfs":{"total":96636764160,"available":96512618496,"internally_reserved":0,"allocated":4198400,"data_stored":2806924,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":119929286},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5861,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:14:55.484 DEBUG:teuthology.orchestra.run.trial098:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-22T20:14:55.646 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T20:14:55.646 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T20:14:55.646 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-22T20:14:55.646 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:14:57.344 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~0s 2026-02-22T20:15:01.748 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-22T20:15:01.748 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-22T20:15:01.748 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:15:01.827 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-22T20:15:01.827 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-22T20:15:01.827 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:15:01.908 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-22T20:15:01.908 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T20:15:01.908 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-22T20:15:01.908 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:15:01.963 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-22T20:15:01.963 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T20:15:01.964 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-22T20:15:01.964 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:15:01.979 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-22T20:15:01.979 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T20:15:01.979 DEBUG:teuthology.orchestra.run.trial098:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T20:15:02.033 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial098.front.sepia.ceph.com 2026-02-22T20:15:02.034 DEBUG:teuthology.orchestra.run.trial098:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-22T20:15:02.128 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial098.front.sepia.ceph.com 2026-02-22T20:15:02.128 DEBUG:teuthology.orchestra.run.trial098:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-22T20:15:02.145 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial098.front.sepia.ceph.com 2026-02-22T20:15:02.145 DEBUG:teuthology.orchestra.run.trial098:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-22T20:15:02.203 INFO:tasks.ceph:Archiving mon data... 2026-02-22T20:15:02.205 DEBUG:teuthology.misc:Transferring archived files from trial098:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002/data/cluster2.mon.a.tgz 2026-02-22T20:15:02.205 DEBUG:teuthology.orchestra.run.trial098:> mktemp 2026-02-22T20:15:02.250 INFO:teuthology.orchestra.run.trial098.stdout:/tmp/tmp.YTd8WuPyMY 2026-02-22T20:15:02.250 DEBUG:teuthology.orchestra.run.trial098:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.YTd8WuPyMY 2026-02-22T20:15:02.345 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 has been restored 2026-02-22T20:15:02.368 DEBUG:teuthology.orchestra.run.trial098:> sudo chmod 0666 /tmp/tmp.YTd8WuPyMY 2026-02-22T20:15:02.429 DEBUG:teuthology.orchestra.remote:trial098:/tmp/tmp.YTd8WuPyMY is 522KB 2026-02-22T20:15:02.537 DEBUG:teuthology.orchestra.run.trial098:> rm -fr /tmp/tmp.YTd8WuPyMY 2026-02-22T20:15:02.541 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T20:15:02.541 DEBUG:teuthology.orchestra.run.trial012:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T20:15:02.543 DEBUG:teuthology.orchestra.run.trial098:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T20:15:02.590 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T20:15:02.607 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T20:15:02.607 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T20:15:02.750 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T20:15:02.751 INFO:teuthology.orchestra.run.trial012.stderr:dumped all 2026-02-22T20:15:02.760 INFO:teuthology.orchestra.run.trial012.stdout:{"pg_ready":true,"pg_map":{"version":1315,"stamp":"2026-02-22T20:15:02.121769+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":296344,"kb_used_data":14172,"kb_used_omap":4,"kb_used_meta":282107,"kb_avail":282819176,"statfs":{"total":289910292480,"available":289606836224,"internally_reserved":0,"allocated":14512128,"data_stored":8746546,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4781,"internal_metadata":288877907},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000867"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995425+0000","last_change":"2026-02-22T19:50:18.919890+0000","last_active":"2026-02-22T20:14:26.995425+0000","last_peered":"2026-02-22T20:14:26.995425+0000","last_clean":"2026-02-22T20:14:26.995425+0000","last_became_active":"2026-02-22T19:50:16.919587+0000","last_became_peered":"2026-02-22T19:50:16.919587+0000","last_unstale":"2026-02-22T20:14:26.995425+0000","last_undegraded":"2026-02-22T20:14:26.995425+0000","last_fullsized":"2026-02-22T20:14:26.995425+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:55:28.698393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389625,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995685+0000","last_change":"2026-02-22T19:50:18.919963+0000","last_active":"2026-02-22T20:14:26.995685+0000","last_peered":"2026-02-22T20:14:26.995685+0000","last_clean":"2026-02-22T20:14:26.995685+0000","last_became_active":"2026-02-22T19:50:16.919517+0000","last_became_peered":"2026-02-22T19:50:16.919517+0000","last_unstale":"2026-02-22T20:14:26.995685+0000","last_undegraded":"2026-02-22T20:14:26.995685+0000","last_fullsized":"2026-02-22T20:14:26.995685+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:48:20.395749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046687900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995899+0000","last_change":"2026-02-22T19:50:18.919761+0000","last_active":"2026-02-22T20:14:26.995899+0000","last_peered":"2026-02-22T20:14:26.995899+0000","last_clean":"2026-02-22T20:14:26.995899+0000","last_became_active":"2026-02-22T19:50:16.918703+0000","last_became_peered":"2026-02-22T19:50:16.918703+0000","last_unstale":"2026-02-22T20:14:26.995899+0000","last_undegraded":"2026-02-22T20:14:26.995899+0000","last_fullsized":"2026-02-22T20:14:26.995899+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:54.246239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027795700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.994396+0000","last_change":"2026-02-22T19:50:18.920056+0000","last_active":"2026-02-22T20:14:26.994396+0000","last_peered":"2026-02-22T20:14:26.994396+0000","last_clean":"2026-02-22T20:14:26.994396+0000","last_became_active":"2026-02-22T19:50:16.919409+0000","last_became_peered":"2026-02-22T19:50:16.919409+0000","last_unstale":"2026-02-22T20:14:26.994396+0000","last_undegraded":"2026-02-22T20:14:26.994396+0000","last_fullsized":"2026-02-22T20:14:26.994396+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:48.980771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053050700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":1142,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.572167+0000","last_change":"2026-02-22T19:50:18.922506+0000","last_active":"2026-02-22T20:14:28.572167+0000","last_peered":"2026-02-22T20:14:28.572167+0000","last_clean":"2026-02-22T20:14:28.572167+0000","last_became_active":"2026-02-22T19:50:16.918968+0000","last_became_peered":"2026-02-22T19:50:16.918968+0000","last_unstale":"2026-02-22T20:14:28.572167+0000","last_undegraded":"2026-02-22T20:14:28.572167+0000","last_fullsized":"2026-02-22T20:14:28.572167+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:44:51.391464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065213399999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.639838+0000","last_change":"2026-02-22T19:50:19.270698+0000","last_active":"2026-02-22T20:14:28.639838+0000","last_peered":"2026-02-22T20:14:28.639838+0000","last_clean":"2026-02-22T20:14:28.639838+0000","last_became_active":"2026-02-22T19:50:16.919079+0000","last_became_peered":"2026-02-22T19:50:16.919079+0000","last_unstale":"2026-02-22T20:14:28.639838+0000","last_undegraded":"2026-02-22T20:14:28.639838+0000","last_fullsized":"2026-02-22T20:14:28.639838+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T20:18:59.837206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028262600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1141,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.639062+0000","last_change":"2026-02-22T19:50:19.270701+0000","last_active":"2026-02-22T20:14:28.639062+0000","last_peered":"2026-02-22T20:14:28.639062+0000","last_clean":"2026-02-22T20:14:28.639062+0000","last_became_active":"2026-02-22T19:50:16.919023+0000","last_became_peered":"2026-02-22T19:50:16.919023+0000","last_unstale":"2026-02-22T20:14:28.639062+0000","last_undegraded":"2026-02-22T20:14:28.639062+0000","last_fullsized":"2026-02-22T20:14:28.639062+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:29:16.953662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033472299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":1141,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.994970+0000","last_change":"2026-02-22T19:50:18.919746+0000","last_active":"2026-02-22T20:14:26.994970+0000","last_peered":"2026-02-22T20:14:26.994970+0000","last_clean":"2026-02-22T20:14:26.994970+0000","last_became_active":"2026-02-22T19:50:16.918296+0000","last_became_peered":"2026-02-22T19:50:16.918296+0000","last_unstale":"2026-02-22T20:14:26.994970+0000","last_undegraded":"2026-02-22T20:14:26.994970+0000","last_fullsized":"2026-02-22T20:14:26.994970+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:31:42.053515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026312999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":1231,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995836+0000","last_change":"2026-02-22T19:50:14.235773+0000","last_active":"2026-02-22T20:14:26.995836+0000","last_peered":"2026-02-22T20:14:26.995836+0000","last_clean":"2026-02-22T20:14:26.995836+0000","last_became_active":"2026-02-22T19:50:14.235079+0000","last_became_peered":"2026-02-22T19:50:14.235079+0000","last_unstale":"2026-02-22T20:14:26.995836+0000","last_undegraded":"2026-02-22T20:14:26.995836+0000","last_fullsized":"2026-02-22T20:14:26.995836+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_clean_scrub_stamp":"2026-02-22T19:50:12.905907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:02:57.829873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705966,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89952,"kb_used_data":4684,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281888,"statfs":{"total":96636764160,"available":96544653312,"internally_reserved":0,"allocated":4796416,"data_stored":2521924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705964,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108060,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263780,"statfs":{"total":96636764160,"available":96526110720,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":105773510},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738668,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":98332,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":93566,"kb_avail":94273508,"statfs":{"total":96636764160,"available":96536072192,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":95812038},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:15:02.762 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T20:15:02.912 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T20:15:02.913 INFO:teuthology.orchestra.run.trial012.stderr:dumped all 2026-02-22T20:15:02.923 INFO:teuthology.orchestra.run.trial012.stdout:{"pg_ready":true,"pg_map":{"version":1315,"stamp":"2026-02-22T20:15:02.121769+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":296344,"kb_used_data":14172,"kb_used_omap":4,"kb_used_meta":282107,"kb_avail":282819176,"statfs":{"total":289910292480,"available":289606836224,"internally_reserved":0,"allocated":14512128,"data_stored":8746546,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4781,"internal_metadata":288877907},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000867"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995425+0000","last_change":"2026-02-22T19:50:18.919890+0000","last_active":"2026-02-22T20:14:26.995425+0000","last_peered":"2026-02-22T20:14:26.995425+0000","last_clean":"2026-02-22T20:14:26.995425+0000","last_became_active":"2026-02-22T19:50:16.919587+0000","last_became_peered":"2026-02-22T19:50:16.919587+0000","last_unstale":"2026-02-22T20:14:26.995425+0000","last_undegraded":"2026-02-22T20:14:26.995425+0000","last_fullsized":"2026-02-22T20:14:26.995425+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:55:28.698393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389625,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995685+0000","last_change":"2026-02-22T19:50:18.919963+0000","last_active":"2026-02-22T20:14:26.995685+0000","last_peered":"2026-02-22T20:14:26.995685+0000","last_clean":"2026-02-22T20:14:26.995685+0000","last_became_active":"2026-02-22T19:50:16.919517+0000","last_became_peered":"2026-02-22T19:50:16.919517+0000","last_unstale":"2026-02-22T20:14:26.995685+0000","last_undegraded":"2026-02-22T20:14:26.995685+0000","last_fullsized":"2026-02-22T20:14:26.995685+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:48:20.395749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046687900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995899+0000","last_change":"2026-02-22T19:50:18.919761+0000","last_active":"2026-02-22T20:14:26.995899+0000","last_peered":"2026-02-22T20:14:26.995899+0000","last_clean":"2026-02-22T20:14:26.995899+0000","last_became_active":"2026-02-22T19:50:16.918703+0000","last_became_peered":"2026-02-22T19:50:16.918703+0000","last_unstale":"2026-02-22T20:14:26.995899+0000","last_undegraded":"2026-02-22T20:14:26.995899+0000","last_fullsized":"2026-02-22T20:14:26.995899+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:54.246239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027795700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.994396+0000","last_change":"2026-02-22T19:50:18.920056+0000","last_active":"2026-02-22T20:14:26.994396+0000","last_peered":"2026-02-22T20:14:26.994396+0000","last_clean":"2026-02-22T20:14:26.994396+0000","last_became_active":"2026-02-22T19:50:16.919409+0000","last_became_peered":"2026-02-22T19:50:16.919409+0000","last_unstale":"2026-02-22T20:14:26.994396+0000","last_undegraded":"2026-02-22T20:14:26.994396+0000","last_fullsized":"2026-02-22T20:14:26.994396+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:48.980771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053050700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":1142,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.572167+0000","last_change":"2026-02-22T19:50:18.922506+0000","last_active":"2026-02-22T20:14:28.572167+0000","last_peered":"2026-02-22T20:14:28.572167+0000","last_clean":"2026-02-22T20:14:28.572167+0000","last_became_active":"2026-02-22T19:50:16.918968+0000","last_became_peered":"2026-02-22T19:50:16.918968+0000","last_unstale":"2026-02-22T20:14:28.572167+0000","last_undegraded":"2026-02-22T20:14:28.572167+0000","last_fullsized":"2026-02-22T20:14:28.572167+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:44:51.391464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065213399999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.639838+0000","last_change":"2026-02-22T19:50:19.270698+0000","last_active":"2026-02-22T20:14:28.639838+0000","last_peered":"2026-02-22T20:14:28.639838+0000","last_clean":"2026-02-22T20:14:28.639838+0000","last_became_active":"2026-02-22T19:50:16.919079+0000","last_became_peered":"2026-02-22T19:50:16.919079+0000","last_unstale":"2026-02-22T20:14:28.639838+0000","last_undegraded":"2026-02-22T20:14:28.639838+0000","last_fullsized":"2026-02-22T20:14:28.639838+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T20:18:59.837206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028262600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1141,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.639062+0000","last_change":"2026-02-22T19:50:19.270701+0000","last_active":"2026-02-22T20:14:28.639062+0000","last_peered":"2026-02-22T20:14:28.639062+0000","last_clean":"2026-02-22T20:14:28.639062+0000","last_became_active":"2026-02-22T19:50:16.919023+0000","last_became_peered":"2026-02-22T19:50:16.919023+0000","last_unstale":"2026-02-22T20:14:28.639062+0000","last_undegraded":"2026-02-22T20:14:28.639062+0000","last_fullsized":"2026-02-22T20:14:28.639062+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:29:16.953662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033472299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":1141,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.994970+0000","last_change":"2026-02-22T19:50:18.919746+0000","last_active":"2026-02-22T20:14:26.994970+0000","last_peered":"2026-02-22T20:14:26.994970+0000","last_clean":"2026-02-22T20:14:26.994970+0000","last_became_active":"2026-02-22T19:50:16.918296+0000","last_became_peered":"2026-02-22T19:50:16.918296+0000","last_unstale":"2026-02-22T20:14:26.994970+0000","last_undegraded":"2026-02-22T20:14:26.994970+0000","last_fullsized":"2026-02-22T20:14:26.994970+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:31:42.053515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026312999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":1231,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995836+0000","last_change":"2026-02-22T19:50:14.235773+0000","last_active":"2026-02-22T20:14:26.995836+0000","last_peered":"2026-02-22T20:14:26.995836+0000","last_clean":"2026-02-22T20:14:26.995836+0000","last_became_active":"2026-02-22T19:50:14.235079+0000","last_became_peered":"2026-02-22T19:50:14.235079+0000","last_unstale":"2026-02-22T20:14:26.995836+0000","last_undegraded":"2026-02-22T20:14:26.995836+0000","last_fullsized":"2026-02-22T20:14:26.995836+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_clean_scrub_stamp":"2026-02-22T19:50:12.905907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:02:57.829873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705966,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89952,"kb_used_data":4684,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281888,"statfs":{"total":96636764160,"available":96544653312,"internally_reserved":0,"allocated":4796416,"data_stored":2521924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705964,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108060,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263780,"statfs":{"total":96636764160,"available":96526110720,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":105773510},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738668,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":98332,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":93566,"kb_avail":94273508,"statfs":{"total":96636764160,"available":96536072192,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":95812038},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:15:02.924 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T20:15:02.924 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T20:15:03.077 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T20:15:03.078 INFO:teuthology.orchestra.run.trial012.stderr:dumped all 2026-02-22T20:15:03.090 INFO:teuthology.orchestra.run.trial012.stdout:{"pg_ready":true,"pg_map":{"version":1315,"stamp":"2026-02-22T20:15:02.121769+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":296344,"kb_used_data":14172,"kb_used_omap":4,"kb_used_meta":282107,"kb_avail":282819176,"statfs":{"total":289910292480,"available":289606836224,"internally_reserved":0,"allocated":14512128,"data_stored":8746546,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4781,"internal_metadata":288877907},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000867"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995425+0000","last_change":"2026-02-22T19:50:18.919890+0000","last_active":"2026-02-22T20:14:26.995425+0000","last_peered":"2026-02-22T20:14:26.995425+0000","last_clean":"2026-02-22T20:14:26.995425+0000","last_became_active":"2026-02-22T19:50:16.919587+0000","last_became_peered":"2026-02-22T19:50:16.919587+0000","last_unstale":"2026-02-22T20:14:26.995425+0000","last_undegraded":"2026-02-22T20:14:26.995425+0000","last_fullsized":"2026-02-22T20:14:26.995425+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:55:28.698393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389625,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995685+0000","last_change":"2026-02-22T19:50:18.919963+0000","last_active":"2026-02-22T20:14:26.995685+0000","last_peered":"2026-02-22T20:14:26.995685+0000","last_clean":"2026-02-22T20:14:26.995685+0000","last_became_active":"2026-02-22T19:50:16.919517+0000","last_became_peered":"2026-02-22T19:50:16.919517+0000","last_unstale":"2026-02-22T20:14:26.995685+0000","last_undegraded":"2026-02-22T20:14:26.995685+0000","last_fullsized":"2026-02-22T20:14:26.995685+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:48:20.395749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046687900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995899+0000","last_change":"2026-02-22T19:50:18.919761+0000","last_active":"2026-02-22T20:14:26.995899+0000","last_peered":"2026-02-22T20:14:26.995899+0000","last_clean":"2026-02-22T20:14:26.995899+0000","last_became_active":"2026-02-22T19:50:16.918703+0000","last_became_peered":"2026-02-22T19:50:16.918703+0000","last_unstale":"2026-02-22T20:14:26.995899+0000","last_undegraded":"2026-02-22T20:14:26.995899+0000","last_fullsized":"2026-02-22T20:14:26.995899+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:54.246239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027795700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.994396+0000","last_change":"2026-02-22T19:50:18.920056+0000","last_active":"2026-02-22T20:14:26.994396+0000","last_peered":"2026-02-22T20:14:26.994396+0000","last_clean":"2026-02-22T20:14:26.994396+0000","last_became_active":"2026-02-22T19:50:16.919409+0000","last_became_peered":"2026-02-22T19:50:16.919409+0000","last_unstale":"2026-02-22T20:14:26.994396+0000","last_undegraded":"2026-02-22T20:14:26.994396+0000","last_fullsized":"2026-02-22T20:14:26.994396+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:48.980771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053050700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":1142,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.572167+0000","last_change":"2026-02-22T19:50:18.922506+0000","last_active":"2026-02-22T20:14:28.572167+0000","last_peered":"2026-02-22T20:14:28.572167+0000","last_clean":"2026-02-22T20:14:28.572167+0000","last_became_active":"2026-02-22T19:50:16.918968+0000","last_became_peered":"2026-02-22T19:50:16.918968+0000","last_unstale":"2026-02-22T20:14:28.572167+0000","last_undegraded":"2026-02-22T20:14:28.572167+0000","last_fullsized":"2026-02-22T20:14:28.572167+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:44:51.391464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065213399999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.639838+0000","last_change":"2026-02-22T19:50:19.270698+0000","last_active":"2026-02-22T20:14:28.639838+0000","last_peered":"2026-02-22T20:14:28.639838+0000","last_clean":"2026-02-22T20:14:28.639838+0000","last_became_active":"2026-02-22T19:50:16.919079+0000","last_became_peered":"2026-02-22T19:50:16.919079+0000","last_unstale":"2026-02-22T20:14:28.639838+0000","last_undegraded":"2026-02-22T20:14:28.639838+0000","last_fullsized":"2026-02-22T20:14:28.639838+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T20:18:59.837206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028262600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1141,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.639062+0000","last_change":"2026-02-22T19:50:19.270701+0000","last_active":"2026-02-22T20:14:28.639062+0000","last_peered":"2026-02-22T20:14:28.639062+0000","last_clean":"2026-02-22T20:14:28.639062+0000","last_became_active":"2026-02-22T19:50:16.919023+0000","last_became_peered":"2026-02-22T19:50:16.919023+0000","last_unstale":"2026-02-22T20:14:28.639062+0000","last_undegraded":"2026-02-22T20:14:28.639062+0000","last_fullsized":"2026-02-22T20:14:28.639062+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:29:16.953662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033472299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":1141,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.994970+0000","last_change":"2026-02-22T19:50:18.919746+0000","last_active":"2026-02-22T20:14:26.994970+0000","last_peered":"2026-02-22T20:14:26.994970+0000","last_clean":"2026-02-22T20:14:26.994970+0000","last_became_active":"2026-02-22T19:50:16.918296+0000","last_became_peered":"2026-02-22T19:50:16.918296+0000","last_unstale":"2026-02-22T20:14:26.994970+0000","last_undegraded":"2026-02-22T20:14:26.994970+0000","last_fullsized":"2026-02-22T20:14:26.994970+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:31:42.053515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026312999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":1231,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995836+0000","last_change":"2026-02-22T19:50:14.235773+0000","last_active":"2026-02-22T20:14:26.995836+0000","last_peered":"2026-02-22T20:14:26.995836+0000","last_clean":"2026-02-22T20:14:26.995836+0000","last_became_active":"2026-02-22T19:50:14.235079+0000","last_became_peered":"2026-02-22T19:50:14.235079+0000","last_unstale":"2026-02-22T20:14:26.995836+0000","last_undegraded":"2026-02-22T20:14:26.995836+0000","last_fullsized":"2026-02-22T20:14:26.995836+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_clean_scrub_stamp":"2026-02-22T19:50:12.905907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:02:57.829873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705966,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89952,"kb_used_data":4684,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281888,"statfs":{"total":96636764160,"available":96544653312,"internally_reserved":0,"allocated":4796416,"data_stored":2521924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705964,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108060,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263780,"statfs":{"total":96636764160,"available":96526110720,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":105773510},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738668,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":98332,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":93566,"kb_avail":94273508,"statfs":{"total":96636764160,"available":96536072192,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":95812038},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:15:03.091 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T20:15:03.243 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T20:15:03.243 INFO:teuthology.orchestra.run.trial012.stdout:{"epoch":572,"fsid":"1fbaf3f1-4414-4c85-a76d-328a2a8144bc","created":"2026-02-22T19:50:01.737667+0000","modified":"2026-02-22T20:14:26.985858+0000","last_up_change":"2026-02-22T19:50:10.899363+0000","last_in_change":"2026-02-22T19:50:02.307709+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T19:50:11.921200+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T19:50:15.900092+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"918567e3-88de-4556-bdb6-f74f987b785a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6818","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6819","nonce":4046618351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6820","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6821","nonce":4046618351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6824","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6825","nonce":4046618351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6822","nonce":4046618351},{"type":"v1","addr":"10.20.193.12:6823","nonce":4046618351}]},"public_addr":"10.20.193.12:6819/4046618351","cluster_addr":"10.20.193.12:6821/4046618351","heartbeat_back_addr":"10.20.193.12:6825/4046618351","heartbeat_front_addr":"10.20.193.12:6823/4046618351","state":["exists","up"]},{"osd":1,"uuid":"13b0e3fb-8b1f-4417-b735-cd1accb01bc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6802","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6803","nonce":1224390038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6804","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6805","nonce":1224390038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6808","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6809","nonce":1224390038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6806","nonce":1224390038},{"type":"v1","addr":"10.20.193.12:6807","nonce":1224390038}]},"public_addr":"10.20.193.12:6803/1224390038","cluster_addr":"10.20.193.12:6805/1224390038","heartbeat_back_addr":"10.20.193.12:6809/1224390038","heartbeat_front_addr":"10.20.193.12:6807/1224390038","state":["exists","up"]},{"osd":2,"uuid":"079aba72-a855-44a4-9127-c34c8eafdc06","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6810","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6811","nonce":1321751267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6812","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6813","nonce":1321751267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6816","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6817","nonce":1321751267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.12:6814","nonce":1321751267},{"type":"v1","addr":"10.20.193.12:6815","nonce":1321751267}]},"public_addr":"10.20.193.12:6811/1321751267","cluster_addr":"10.20.193.12:6813/1321751267","heartbeat_back_addr":"10.20.193.12:6817/1321751267","heartbeat_front_addr":"10.20.193.12:6815/1321751267","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:09.575529+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:08.778690+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T19:50:09.480453+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.98:0/1259162719":"2026-02-22T21:11:28.145276+0000","10.20.193.98:0/116985803":"2026-02-22T21:13:35.349428+0000","10.20.193.98:0/2214767580":"2026-02-22T21:14:05.956806+0000","10.20.193.98:0/4287057391":"2026-02-22T21:11:28.390811+0000","10.20.193.98:0/171864836":"2026-02-22T21:11:58.859604+0000","10.20.193.98:0/622723143":"2026-02-22T21:14:05.956598+0000","10.20.193.98:0/3893948960":"2026-02-22T21:11:58.859374+0000","10.20.193.98:0/2378133346":"2026-02-22T21:11:58.860516+0000","10.20.193.98:0/230742933":"2026-02-22T21:11:58.859910+0000","10.20.193.98:0/2226579375":"2026-02-22T21:11:58.860193+0000","10.20.193.98:0/2109007344":"2026-02-22T21:14:05.956239+0000","10.20.193.98:0/1943255863":"2026-02-22T21:14:05.957016+0000","10.20.193.98:0/4176888753":"2026-02-22T21:11:58.860364+0000","10.20.193.98:0/553922254":"2026-02-22T21:14:05.956702+0000","10.20.193.98:0/3021394402":"2026-02-22T21:13:35.100273+0000","10.20.193.98:0/3017547724":"2026-02-22T21:14:05.956913+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":209,"length":1},{"begin":212,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T20:15:04.254 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-22T20:15:04.254 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T20:15:04.330 INFO:teuthology.orchestra.run.trial012.stdout:{ 2026-02-22T20:15:04.330 INFO:teuthology.orchestra.run.trial012.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T20:15:04.330 INFO:teuthology.orchestra.run.trial012.stdout:} 2026-02-22T20:15:04.339 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-22T20:15:04.488 INFO:teuthology.orchestra.run.trial012.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T20:15:04.499 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-22T20:15:04.499 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T20:15:04.581 INFO:teuthology.orchestra.run.trial012.stdout:{ 2026-02-22T20:15:04.581 INFO:teuthology.orchestra.run.trial012.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T20:15:04.582 INFO:teuthology.orchestra.run.trial012.stdout:} 2026-02-22T20:15:04.590 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-22T20:15:04.731 INFO:teuthology.orchestra.run.trial012.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T20:15:04.742 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-22T20:15:04.742 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T20:15:04.814 INFO:teuthology.orchestra.run.trial012.stdout:{ 2026-02-22T20:15:04.814 INFO:teuthology.orchestra.run.trial012.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T20:15:04.814 INFO:teuthology.orchestra.run.trial012.stdout:} 2026-02-22T20:15:04.823 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-22T20:15:04.971 INFO:teuthology.orchestra.run.trial012.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T20:15:04.982 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T20:15:05.135 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T20:15:05.136 INFO:teuthology.orchestra.run.trial012.stderr:dumped all 2026-02-22T20:15:05.146 INFO:teuthology.orchestra.run.trial012.stdout:{"pg_ready":true,"pg_map":{"version":1316,"stamp":"2026-02-22T20:15:04.121961+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":296344,"kb_used_data":14172,"kb_used_omap":4,"kb_used_meta":282107,"kb_avail":282819176,"statfs":{"total":289910292480,"available":289606836224,"internally_reserved":0,"allocated":14512128,"data_stored":8746546,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4781,"internal_metadata":288877907},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000913"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995425+0000","last_change":"2026-02-22T19:50:18.919890+0000","last_active":"2026-02-22T20:14:26.995425+0000","last_peered":"2026-02-22T20:14:26.995425+0000","last_clean":"2026-02-22T20:14:26.995425+0000","last_became_active":"2026-02-22T19:50:16.919587+0000","last_became_peered":"2026-02-22T19:50:16.919587+0000","last_unstale":"2026-02-22T20:14:26.995425+0000","last_undegraded":"2026-02-22T20:14:26.995425+0000","last_fullsized":"2026-02-22T20:14:26.995425+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:55:28.698393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389625,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995685+0000","last_change":"2026-02-22T19:50:18.919963+0000","last_active":"2026-02-22T20:14:26.995685+0000","last_peered":"2026-02-22T20:14:26.995685+0000","last_clean":"2026-02-22T20:14:26.995685+0000","last_became_active":"2026-02-22T19:50:16.919517+0000","last_became_peered":"2026-02-22T19:50:16.919517+0000","last_unstale":"2026-02-22T20:14:26.995685+0000","last_undegraded":"2026-02-22T20:14:26.995685+0000","last_fullsized":"2026-02-22T20:14:26.995685+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:48:20.395749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046687900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995899+0000","last_change":"2026-02-22T19:50:18.919761+0000","last_active":"2026-02-22T20:14:26.995899+0000","last_peered":"2026-02-22T20:14:26.995899+0000","last_clean":"2026-02-22T20:14:26.995899+0000","last_became_active":"2026-02-22T19:50:16.918703+0000","last_became_peered":"2026-02-22T19:50:16.918703+0000","last_unstale":"2026-02-22T20:14:26.995899+0000","last_undegraded":"2026-02-22T20:14:26.995899+0000","last_fullsized":"2026-02-22T20:14:26.995899+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:54.246239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027795700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.994396+0000","last_change":"2026-02-22T19:50:18.920056+0000","last_active":"2026-02-22T20:14:26.994396+0000","last_peered":"2026-02-22T20:14:26.994396+0000","last_clean":"2026-02-22T20:14:26.994396+0000","last_became_active":"2026-02-22T19:50:16.919409+0000","last_became_peered":"2026-02-22T19:50:16.919409+0000","last_unstale":"2026-02-22T20:14:26.994396+0000","last_undegraded":"2026-02-22T20:14:26.994396+0000","last_fullsized":"2026-02-22T20:14:26.994396+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:48.980771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053050700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":1142,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.572167+0000","last_change":"2026-02-22T19:50:18.922506+0000","last_active":"2026-02-22T20:14:28.572167+0000","last_peered":"2026-02-22T20:14:28.572167+0000","last_clean":"2026-02-22T20:14:28.572167+0000","last_became_active":"2026-02-22T19:50:16.918968+0000","last_became_peered":"2026-02-22T19:50:16.918968+0000","last_unstale":"2026-02-22T20:14:28.572167+0000","last_undegraded":"2026-02-22T20:14:28.572167+0000","last_fullsized":"2026-02-22T20:14:28.572167+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:44:51.391464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065213399999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1140,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.639838+0000","last_change":"2026-02-22T19:50:19.270698+0000","last_active":"2026-02-22T20:14:28.639838+0000","last_peered":"2026-02-22T20:14:28.639838+0000","last_clean":"2026-02-22T20:14:28.639838+0000","last_became_active":"2026-02-22T19:50:16.919079+0000","last_became_peered":"2026-02-22T19:50:16.919079+0000","last_unstale":"2026-02-22T20:14:28.639838+0000","last_undegraded":"2026-02-22T20:14:28.639838+0000","last_fullsized":"2026-02-22T20:14:28.639838+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T20:18:59.837206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028262600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1141,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:28.639062+0000","last_change":"2026-02-22T19:50:19.270701+0000","last_active":"2026-02-22T20:14:28.639062+0000","last_peered":"2026-02-22T20:14:28.639062+0000","last_clean":"2026-02-22T20:14:28.639062+0000","last_became_active":"2026-02-22T19:50:16.919023+0000","last_became_peered":"2026-02-22T19:50:16.919023+0000","last_unstale":"2026-02-22T20:14:28.639062+0000","last_undegraded":"2026-02-22T20:14:28.639062+0000","last_fullsized":"2026-02-22T20:14:28.639062+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:29:16.953662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033472299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":1141,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.994970+0000","last_change":"2026-02-22T19:50:18.919746+0000","last_active":"2026-02-22T20:14:26.994970+0000","last_peered":"2026-02-22T20:14:26.994970+0000","last_clean":"2026-02-22T20:14:26.994970+0000","last_became_active":"2026-02-22T19:50:16.918296+0000","last_became_peered":"2026-02-22T19:50:16.918296+0000","last_unstale":"2026-02-22T20:14:26.994970+0000","last_undegraded":"2026-02-22T20:14:26.994970+0000","last_fullsized":"2026-02-22T20:14:26.994970+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:15.910688+0000","last_clean_scrub_stamp":"2026-02-22T19:50:15.910688+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:31:42.053515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026312999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":1231,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:14:26.995836+0000","last_change":"2026-02-22T19:50:14.235773+0000","last_active":"2026-02-22T20:14:26.995836+0000","last_peered":"2026-02-22T20:14:26.995836+0000","last_clean":"2026-02-22T20:14:26.995836+0000","last_became_active":"2026-02-22T19:50:14.235079+0000","last_became_peered":"2026-02-22T19:50:14.235079+0000","last_unstale":"2026-02-22T20:14:26.995836+0000","last_undegraded":"2026-02-22T20:14:26.995836+0000","last_fullsized":"2026-02-22T20:14:26.995836+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T19:50:12.905907+0000","last_clean_scrub_stamp":"2026-02-22T19:50:12.905907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:02:57.829873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705966,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89952,"kb_used_data":4684,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281888,"statfs":{"total":96636764160,"available":96544653312,"internally_reserved":0,"allocated":4796416,"data_stored":2521924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705964,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108060,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263780,"statfs":{"total":96636764160,"available":96526110720,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":105773510},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738668,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":98332,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":93566,"kb_avail":94273508,"statfs":{"total":96636764160,"available":96536072192,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":95812038},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:15:05.147 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T19:50:15.910688+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=15, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=15, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:15:05.148 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T19:50:15.910688+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=15, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=15, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:15:05.148 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T19:50:15.910688+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=15, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=15, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:15:05.148 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T19:50:15.910688+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=15, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=15, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:15:05.148 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T19:50:15.910688+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=15, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=15, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:15:05.148 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T19:50:15.910688+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=15, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=15, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:15:05.149 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T19:50:15.910688+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=15, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=15, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:15:05.149 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T19:50:15.910688+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=15, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=15, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:15:05.149 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T19:50:12.905907+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=19, tm_min=50, tm_sec=12, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=20, tm_min=15, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T20:15:05.149 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T20:15:25.149 DEBUG:teuthology.orchestra.run.trial012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T20:15:25.304 INFO:teuthology.orchestra.run.trial012.stdout: 2026-02-22T20:15:25.305 INFO:teuthology.orchestra.run.trial012.stderr:dumped all 2026-02-22T20:15:25.314 INFO:teuthology.orchestra.run.trial012.stdout:{"pg_ready":true,"pg_map":{"version":1326,"stamp":"2026-02-22T20:15:24.123722+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":296332,"kb_used_data":14172,"kb_used_omap":4,"kb_used_meta":282107,"kb_avail":282819188,"statfs":{"total":289910292480,"available":289606848512,"internally_reserved":0,"allocated":14512128,"data_stored":8746546,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4781,"internal_metadata":288877907},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000958"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1149,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:15:08.831021+0000","last_change":"2026-02-22T20:15:08.831039+0000","last_active":"2026-02-22T20:15:08.831021+0000","last_peered":"2026-02-22T20:15:08.831021+0000","last_clean":"2026-02-22T20:15:08.831021+0000","last_became_active":"2026-02-22T19:50:16.919587+0000","last_became_peered":"2026-02-22T19:50:16.919587+0000","last_unstale":"2026-02-22T20:15:08.831021+0000","last_undegraded":"2026-02-22T20:15:08.831021+0000","last_fullsized":"2026-02-22T20:15:08.831021+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:15:08.830935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:15:08.830935+0000","last_clean_scrub_stamp":"2026-02-22T20:15:08.830935+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:52:03.679263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389625,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1149,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:15:07.814487+0000","last_change":"2026-02-22T20:15:07.814536+0000","last_active":"2026-02-22T20:15:07.814487+0000","last_peered":"2026-02-22T20:15:07.814487+0000","last_clean":"2026-02-22T20:15:07.814487+0000","last_became_active":"2026-02-22T19:50:16.919517+0000","last_became_peered":"2026-02-22T19:50:16.919517+0000","last_unstale":"2026-02-22T20:15:07.814487+0000","last_undegraded":"2026-02-22T20:15:07.814487+0000","last_fullsized":"2026-02-22T20:15:07.814487+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:15:07.814413+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:15:07.814413+0000","last_clean_scrub_stamp":"2026-02-22T20:15:07.814413+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:32:22.634473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046687900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1149,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:15:06.847569+0000","last_change":"2026-02-22T20:15:06.847600+0000","last_active":"2026-02-22T20:15:06.847569+0000","last_peered":"2026-02-22T20:15:06.847569+0000","last_clean":"2026-02-22T20:15:06.847569+0000","last_became_active":"2026-02-22T19:50:16.918703+0000","last_became_peered":"2026-02-22T19:50:16.918703+0000","last_unstale":"2026-02-22T20:15:06.847569+0000","last_undegraded":"2026-02-22T20:15:06.847569+0000","last_fullsized":"2026-02-22T20:15:06.847569+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:15:06.847502+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:15:06.847502+0000","last_clean_scrub_stamp":"2026-02-22T20:15:06.847502+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:18:23.474428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027795700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1150,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:15:09.814279+0000","last_change":"2026-02-22T20:15:09.814279+0000","last_active":"2026-02-22T20:15:09.814279+0000","last_peered":"2026-02-22T20:15:09.814279+0000","last_clean":"2026-02-22T20:15:09.814279+0000","last_became_active":"2026-02-22T19:50:16.919409+0000","last_became_peered":"2026-02-22T19:50:16.919409+0000","last_unstale":"2026-02-22T20:15:09.814279+0000","last_undegraded":"2026-02-22T20:15:09.814279+0000","last_fullsized":"2026-02-22T20:15:09.814279+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:15:09.814188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:15:09.814188+0000","last_clean_scrub_stamp":"2026-02-22T20:15:09.814188+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:43:12.861289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053050700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"572'3","reported_seq":1153,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:15:05.141144+0000","last_change":"2026-02-22T20:15:05.141144+0000","last_active":"2026-02-22T20:15:05.141144+0000","last_peered":"2026-02-22T20:15:05.141144+0000","last_clean":"2026-02-22T20:15:05.141144+0000","last_became_active":"2026-02-22T19:50:16.918968+0000","last_became_peered":"2026-02-22T19:50:16.918968+0000","last_unstale":"2026-02-22T20:15:05.141144+0000","last_undegraded":"2026-02-22T20:15:05.141144+0000","last_fullsized":"2026-02-22T20:15:05.141144+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"572'3","last_scrub_stamp":"2026-02-22T20:15:05.141009+0000","last_deep_scrub":"572'3","last_deep_scrub_stamp":"2026-02-22T20:15:05.141009+0000","last_clean_scrub_stamp":"2026-02-22T20:15:05.141009+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:56:10.064060+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00065213399999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1150,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:15:06.414822+0000","last_change":"2026-02-22T20:15:06.414822+0000","last_active":"2026-02-22T20:15:06.414822+0000","last_peered":"2026-02-22T20:15:06.414822+0000","last_clean":"2026-02-22T20:15:06.414822+0000","last_became_active":"2026-02-22T19:50:16.919079+0000","last_became_peered":"2026-02-22T19:50:16.919079+0000","last_unstale":"2026-02-22T20:15:06.414822+0000","last_undegraded":"2026-02-22T20:15:06.414822+0000","last_fullsized":"2026-02-22T20:15:06.414822+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:15:06.414737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:15:06.414737+0000","last_clean_scrub_stamp":"2026-02-22T20:15:06.414737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:45:46.904450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028262600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1150,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:15:05.430323+0000","last_change":"2026-02-22T20:15:05.430337+0000","last_active":"2026-02-22T20:15:05.430323+0000","last_peered":"2026-02-22T20:15:05.430323+0000","last_clean":"2026-02-22T20:15:05.430323+0000","last_became_active":"2026-02-22T19:50:16.919023+0000","last_became_peered":"2026-02-22T19:50:16.919023+0000","last_unstale":"2026-02-22T20:15:05.430323+0000","last_undegraded":"2026-02-22T20:15:05.430323+0000","last_fullsized":"2026-02-22T20:15:05.430323+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T20:15:05.430269+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T20:15:05.430269+0000","last_clean_scrub_stamp":"2026-02-22T20:15:05.430269+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:26:39.453975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033472299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"572'2","reported_seq":1152,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:15:05.822114+0000","last_change":"2026-02-22T20:15:05.822114+0000","last_active":"2026-02-22T20:15:05.822114+0000","last_peered":"2026-02-22T20:15:05.822114+0000","last_clean":"2026-02-22T20:15:05.822114+0000","last_became_active":"2026-02-22T19:50:16.918296+0000","last_became_peered":"2026-02-22T19:50:16.918296+0000","last_unstale":"2026-02-22T20:15:05.822114+0000","last_undegraded":"2026-02-22T20:15:05.822114+0000","last_fullsized":"2026-02-22T20:15:05.822114+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"572'2","last_scrub_stamp":"2026-02-22T20:15:05.822047+0000","last_deep_scrub":"572'2","last_deep_scrub_stamp":"2026-02-22T20:15:05.822047+0000","last_clean_scrub_stamp":"2026-02-22T20:15:05.822047+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:08:06.488546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026312999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"572'74","reported_seq":1242,"reported_epoch":572,"state":"active+clean","last_fresh":"2026-02-22T20:15:04.838109+0000","last_change":"2026-02-22T20:15:04.838143+0000","last_active":"2026-02-22T20:15:04.838109+0000","last_peered":"2026-02-22T20:15:04.838109+0000","last_clean":"2026-02-22T20:15:04.838109+0000","last_became_active":"2026-02-22T19:50:14.235079+0000","last_became_peered":"2026-02-22T19:50:14.235079+0000","last_unstale":"2026-02-22T20:15:04.838109+0000","last_undegraded":"2026-02-22T20:15:04.838109+0000","last_fullsized":"2026-02-22T20:15:04.838109+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"572'74","last_scrub_stamp":"2026-02-22T20:15:04.838030+0000","last_deep_scrub":"572'74","last_deep_scrub_stamp":"2026-02-22T20:15:04.838030+0000","last_clean_scrub_stamp":"2026-02-22T20:15:04.838030+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:03:25.818585+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11758,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705970,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89948,"kb_used_data":4684,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281892,"statfs":{"total":96636764160,"available":96544657408,"internally_reserved":0,"allocated":4796416,"data_stored":2521924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705968,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108056,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263784,"statfs":{"total":96636764160,"available":96526114816,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":105773510},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738672,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":98328,"kb_used_data":4744,"kb_used_omap":1,"kb_used_meta":93566,"kb_avail":94273512,"statfs":{"total":96636764160,"available":96536076288,"internally_reserved":0,"allocated":4857856,"data_stored":3112311,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":95812038},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5879,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T20:15:25.316 DEBUG:teuthology.orchestra.run.trial012:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-22T20:15:25.477 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T20:15:25.477 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T20:15:25.477 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-22T20:15:25.477 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:15:25.572 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-22T20:15:25.572 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-22T20:15:25.573 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:15:27.302 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 is failed for ~0s 2026-02-22T20:15:31.674 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-22T20:15:31.675 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-22T20:15:31.675 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:15:31.751 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-22T20:15:31.752 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T20:15:31.752 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-22T20:15:31.752 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:15:31.792 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-22T20:15:31.792 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T20:15:31.793 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-22T20:15:31.793 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T20:15:31.807 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-22T20:15:31.807 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T20:15:31.807 DEBUG:teuthology.orchestra.run.trial012:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T20:15:31.862 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial012.front.sepia.ceph.com 2026-02-22T20:15:31.862 DEBUG:teuthology.orchestra.run.trial012:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-22T20:15:31.919 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial012.front.sepia.ceph.com 2026-02-22T20:15:31.920 DEBUG:teuthology.orchestra.run.trial012:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-22T20:15:31.976 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial012.front.sepia.ceph.com 2026-02-22T20:15:31.976 DEBUG:teuthology.orchestra.run.trial012:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-22T20:15:32.030 INFO:tasks.ceph:Archiving mon data... 2026-02-22T20:15:32.031 DEBUG:teuthology.misc:Transferring archived files from trial012:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002/data/cluster1.mon.a.tgz 2026-02-22T20:15:32.031 DEBUG:teuthology.orchestra.run.trial012:> mktemp 2026-02-22T20:15:32.076 INFO:teuthology.orchestra.run.trial012.stdout:/tmp/tmp.HjaZ52RMJ7 2026-02-22T20:15:32.076 DEBUG:teuthology.orchestra.run.trial012:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.HjaZ52RMJ7 2026-02-22T20:15:32.194 DEBUG:teuthology.orchestra.run.trial012:> sudo chmod 0666 /tmp/tmp.HjaZ52RMJ7 2026-02-22T20:15:32.250 DEBUG:teuthology.orchestra.remote:trial012:/tmp/tmp.HjaZ52RMJ7 is 539KB 2026-02-22T20:15:32.302 DEBUG:teuthology.orchestra.run.trial012:> rm -fr /tmp/tmp.HjaZ52RMJ7 2026-02-22T20:15:32.303 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 has been restored 2026-02-22T20:15:32.306 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T20:15:32.306 DEBUG:teuthology.orchestra.run.trial012:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T20:15:32.350 DEBUG:teuthology.orchestra.run.trial098:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T20:15:32.404 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T20:15:32.408 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T20:15:32.408 INFO:tasks.ceph:Archiving crash dumps... 2026-02-22T20:15:32.411 DEBUG:teuthology.misc:Transferring archived files from trial012:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002/remote/trial012/crash 2026-02-22T20:15:32.412 DEBUG:teuthology.orchestra.run.trial012:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T20:15:32.454 DEBUG:teuthology.misc:Transferring archived files from trial098:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002/remote/trial098/crash 2026-02-22T20:15:32.455 DEBUG:teuthology.orchestra.run.trial098:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T20:15:32.463 INFO:tasks.ceph:Compressing logs... 2026-02-22T20:15:32.463 DEBUG:teuthology.orchestra.run.trial012:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T20:15:32.498 DEBUG:teuthology.orchestra.run.trial098:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T20:15:32.505 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21128.log 2026-02-22T20:15:32.506 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21076.log 2026-02-22T20:15:32.506 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.21128.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21128.log.gz 2026-02-22T20:15:32.506 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-22T20:15:32.506 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.21076.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19117.log 2026-02-22T20:15:32.506 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21076.log.gz 2026-02-22T20:15:32.507 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-22T20:15:32.507 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-mon.a.log: /var/log/ceph/cluster1-client.admin.19117.log: gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-22T20:15:32.507 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19117.log.gz 2026-02-22T20:15:32.507 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20946.log 2026-02-22T20:15:32.508 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18962.log 2026-02-22T20:15:32.509 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.20946.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-22T20:15:32.509 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20946.log.gz 2026-02-22T20:15:32.509 INFO:teuthology.orchestra.run.trial012.stderr:gzip/var/log/ceph/cluster1-client.admin.18962.log: -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-22T20:15:32.509 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18962.log.gz 2026-02-22T20:15:32.509 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-22T20:15:32.509 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20868.log 2026-02-22T20:15:32.509 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21050.log 2026-02-22T20:15:32.510 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19012.log 2026-02-22T20:15:32.510 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.20868.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20868.log.gz 2026-02-22T20:15:32.510 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.21050.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21050.log.gz 2026-02-22T20:15:32.510 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19196.log 2026-02-22T20:15:32.510 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19012.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19396.log 2026-02-22T20:15:32.510 INFO:teuthology.orchestra.run.trial012.stderr: 80.6% -- replaced with /var/log/ceph/cluster1-client.admin.19012.log.gz 2026-02-22T20:15:32.511 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19196.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19196.log.gz 2026-02-22T20:15:32.511 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20972.log 2026-02-22T20:15:32.511 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19396.log: gzip -5 --verbose 0.0% -- -- replaced with /var/log/ceph/cluster1-client.admin.19396.log.gz /var/log/ceph/cluster1-client.admin.16231.log 2026-02-22T20:15:32.511 INFO:teuthology.orchestra.run.trial012.stderr: 2026-02-22T20:15:32.511 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.20972.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20920.log 2026-02-22T20:15:32.512 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20972.log.gz 2026-02-22T20:15:32.512 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.16231.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16231.log.gz 2026-02-22T20:15:32.512 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21024.log 2026-02-22T20:15:32.512 INFO:teuthology.orchestra.run.trial012.stderr: 87.0% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-22T20:15:32.512 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.20920.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19058.log 2026-02-22T20:15:32.512 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20920.log.gz 2026-02-22T20:15:32.512 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16330.log 2026-02-22T20:15:32.513 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.21024.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21024.log.gz 2026-02-22T20:15:32.513 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19058.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19058.log.gz 2026-02-22T20:15:32.513 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.12471.log 2026-02-22T20:15:32.513 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.16330.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21154.log 2026-02-22T20:15:32.513 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16330.log.gz 2026-02-22T20:15:32.513 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/ceph.tmp-client.admin.12471.log: 93.8% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-22T20:15:32.513 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19188.log 2026-02-22T20:15:32.514 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.12471.log.gz 2026-02-22T20:15:32.514 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18901.log 2026-02-22T20:15:32.514 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.21154.log: /var/log/ceph/cluster1-client.admin.19188.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21154.log.gz 2026-02-22T20:15:32.514 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19188.log.gz 2026-02-22T20:15:32.514 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19118.log 2026-02-22T20:15:32.514 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20894.log 2026-02-22T20:15:32.514 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.18901.log: /var/log/ceph/cluster1-client.admin.19118.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18901.log.gz 2026-02-22T20:15:32.514 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19118.log.gz 2026-02-22T20:15:32.515 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19266.log 2026-02-22T20:15:32.515 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.20894.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16183.log 2026-02-22T20:15:32.515 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20894.log.gz 2026-02-22T20:15:32.515 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20998.log 2026-02-22T20:15:32.515 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19266.log.gz 2026-02-22T20:15:32.515 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.16183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19273.log 2026-02-22T20:15:32.516 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16183.log.gz 2026-02-22T20:15:32.516 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58795.log 2026-02-22T20:15:32.516 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.20998.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19116.log 2026-02-22T20:15:32.516 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20998.log.gz 2026-02-22T20:15:32.516 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16207.log 2026-02-22T20:15:32.516 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19273.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19273.log.gz 2026-02-22T20:15:32.516 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19116.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19116.log.gz 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18987.log 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.16207.log: /var/log/ceph/cluster1-osd.0.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16207.log.gz 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19370.log 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.18987.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18987.log.gz 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19195.log 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16266.log 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19370.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19370.log.gz 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19195.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19195.log.gz 2026-02-22T20:15:32.518 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19344.log 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.16266.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16266.log.gz 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21178.log 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43469.log 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.58795.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53794.log 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43469.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.58795.log.gz 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41143.log 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.43469.log.gz 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44647.log 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53794.log: /var/log/ceph/cluster2-client.admin.41143.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52251.log 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44647.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.41143.log.gz 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53794.log.gz 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20252.log 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44647.log.gz 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52251.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36669.log 2026-02-22T20:15:32.519 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.20252.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52251.log.gz 2026-02-22T20:15:32.520 INFO:teuthology.orchestra.run.trial098.stderr: 56.4% -- replaced with /var/log/ceph/cluster1-client.admin.20252.log.gz 2026-02-22T20:15:32.520 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44814.log 2026-02-22T20:15:32.520 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.36669.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58699.log 2026-02-22T20:15:32.520 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44814.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.36669.log.gz 2026-02-22T20:15:32.520 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45023.log 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16090.log 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19344.log: /var/log/ceph/cluster1-client.admin.21178.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19344.log.gz 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21178.log.gz 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.16090.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16090.log.gz 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21102.log 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19032.log 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-osd.2.log: /var/log/ceph/cluster1-client.admin.21102.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21102.log.gz 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19084.log 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19274.log 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19032.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19032.log.gz 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr:/var/log/ceph/cluster1-client.admin.19084.log: /var/log/ceph/cluster1-client.admin.19274.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19084.log.gz 2026-02-22T20:15:32.522 INFO:teuthology.orchestra.run.trial012.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19274.log.gz 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.57395.log 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44814.log.gz 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.58699.log: /var/log/ceph/cluster2-client.mirror.45023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38002.log 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58699.log.gz 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58607.log 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.57395.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45023.log.gz 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.38002.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25753.log 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.58607.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.38002.log.gz 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60004.log 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58607.log.gz 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44982.log 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.25753.log: /var/log/ceph/cluster1-client.admin.60004.log: gzip -5 --verbose 0.0% -- -- replaced with /var/log/ceph/cluster1-client.admin.60004.log.gz /var/log/ceph/cluster1-client.mirror.24728.log 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.523 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.25753.log.gz 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44982.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18983.log 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.24728.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.55472.log 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.24728.log.gz 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.18983.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18983.log.gz 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53209.log 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44982.log.gz 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60589.log 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.3.55472.log: /var/log/ceph/cluster2-client.mirror.53209.log: 20.2% -- replaced with /var/log/ceph/cluster2-client.mirror.3.55472.log.gz 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27049.log 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53209.log.gz 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60589.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60589.log.gz 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.22261.log 2026-02-22T20:15:32.524 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.27049.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44316.log 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34628.log 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.27049.log.gz 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.22261.log: /var/log/ceph/cluster2-client.mirror.44316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.55461.log 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.22261.log.gz 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.34628.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45430.log 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.2.55461.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39386.log 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44316.log.gz 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.34628.log.gz 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45430.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23118.log 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50011.log 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.57395.log.gz 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.39386.log: /var/log/ceph/cluster1-client.admin.23118.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45430.log.gz 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39386.log.gz 2026-02-22T20:15:32.525 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51152.log 2026-02-22T20:15:32.526 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.50011.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.23118.log.gz 2026-02-22T20:15:32.526 INFO:teuthology.orchestra.run.trial098.stderr: 80.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50011.log.gz 2026-02-22T20:15:32.526 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34982.log 2026-02-22T20:15:32.526 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.51152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55350.log 2026-02-22T20:15:32.526 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52431.log 2026-02-22T20:15:32.527 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34982.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.51152.log.gz 2026-02-22T20:15:32.527 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.55350.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.51612.log 2026-02-22T20:15:32.527 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55350.log.gz 2026-02-22T20:15:32.527 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.34982.log.gz 2026-02-22T20:15:32.527 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52431.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23325.log 2026-02-22T20:15:32.527 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.51612.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24142.log 2026-02-22T20:15:32.527 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.51612.log.gz 2026-02-22T20:15:32.527 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.52431.log.gz 2026-02-22T20:15:32.528 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23325.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.2.55461.log.gz 2026-02-22T20:15:32.528 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47701.log 2026-02-22T20:15:32.528 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22181.log 2026-02-22T20:15:32.528 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24142.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.23325.log.gz 2026-02-22T20:15:32.528 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40245.log 2026-02-22T20:15:32.528 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47701.log: /var/log/ceph/cluster1-client.admin.22181.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35242.log 2026-02-22T20:15:32.528 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.22181.log.gz 2026-02-22T20:15:32.529 INFO:teuthology.orchestra.run.trial098.stderr: 90.0%gzip -5 -- replaced with /var/log/ceph/cluster2-client.mirror.47701.log.gz --verbose -- 2026-02-22T20:15:32.529 INFO:teuthology.orchestra.run.trial098.stderr: /var/log/ceph/cluster1-client.mirror.2.51640.log 2026-02-22T20:15:32.529 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.40245.log: /var/log/ceph/cluster1-client.mirror.35242.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.24142.log.gz 2026-02-22T20:15:32.529 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24553.log 2026-02-22T20:15:32.529 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.35242.log.gz 2026-02-22T20:15:32.529 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.40245.log.gz 2026-02-22T20:15:32.529 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.51640.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19719.log 2026-02-22T20:15:32.529 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.51640.log.gz 2026-02-22T20:15:32.530 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24553.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42791.log 2026-02-22T20:15:32.530 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.24553.log.gz 2026-02-22T20:15:32.530 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.19719.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34413.log 2026-02-22T20:15:32.530 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19719.log.gz 2026-02-22T20:15:32.530 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.42791.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48222.log 2026-02-22T20:15:32.530 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.42791.log.gz 2026-02-22T20:15:32.530 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53773.log 2026-02-22T20:15:32.531 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.34413.log: /var/log/ceph/cluster2-client.mirror.48222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46748.log 2026-02-22T20:15:32.531 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53773.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38237.log 2026-02-22T20:15:32.531 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.34413.log.gz 2026-02-22T20:15:32.531 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46748.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34260.log 2026-02-22T20:15:32.531 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53773.log.gz 2026-02-22T20:15:32.532 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.38237.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49629.log 2026-02-22T20:15:32.532 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46748.log.gz 2026-02-22T20:15:32.532 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.34260.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25882.log 2026-02-22T20:15:32.532 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38237.log.gz 2026-02-22T20:15:32.532 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34260.log.gz 2026-02-22T20:15:32.532 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.49629.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48014.log 2026-02-22T20:15:32.532 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.49629.log.gz 2026-02-22T20:15:32.532 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27740.log 2026-02-22T20:15:32.533 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.25882.log: /var/log/ceph/cluster1-client.admin.48014.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32275.log 2026-02-22T20:15:32.533 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.25882.log.gz 2026-02-22T20:15:32.533 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.48014.log.gz 2026-02-22T20:15:32.533 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.27740.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30477.log 2026-02-22T20:15:32.533 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32275.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.27740.log.gz 2026-02-22T20:15:32.534 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47942.log 2026-02-22T20:15:32.534 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.30477.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34548.log 2026-02-22T20:15:32.534 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.32275.log.gz 2026-02-22T20:15:32.534 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster2-client.mirror.47942.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22075.log 2026-02-22T20:15:32.534 INFO:teuthology.orchestra.run.trial098.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48222.log.gz 2026-02-22T20:15:32.534 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.30477.log.gz 2026-02-22T20:15:32.534 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34548.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47942.log.gz 2026-02-22T20:15:32.535 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51266.log 2026-02-22T20:15:32.535 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.34548.log.gz 2026-02-22T20:15:32.535 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.22075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56126.log 2026-02-22T20:15:32.535 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.22075.log.gz 2026-02-22T20:15:32.535 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47680.log 2026-02-22T20:15:32.535 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.51266.log: /var/log/ceph/cluster2-client.mirror.56126.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42601.log 2026-02-22T20:15:32.536 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47680.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56126.log.gz 2026-02-22T20:15:32.536 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22866.log 2026-02-22T20:15:32.536 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47680.log.gz 2026-02-22T20:15:32.536 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.42601.log: -- /var/log/ceph/cluster2-client.admin.19188.log 2026-02-22T20:15:32.536 INFO:teuthology.orchestra.run.trial098.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51266.log.gz 2026-02-22T20:15:32.536 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22866.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42601.log.gz 2026-02-22T20:15:32.536 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54433.log 2026-02-22T20:15:32.537 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19188.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52602.log 2026-02-22T20:15:32.537 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.22866.log.gz 0.0% 2026-02-22T20:15:32.537 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.19188.log.gz 2026-02-22T20:15:32.537 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44962.log 2026-02-22T20:15:32.537 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.54433.log: /var/log/ceph/cluster1-client.mirror.52602.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52097.log 2026-02-22T20:15:32.537 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54433.log.gz 2026-02-22T20:15:32.537 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52602.log.gz 2026-02-22T20:15:32.538 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44962.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.51723.log 2026-02-22T20:15:32.538 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26636.log 2026-02-22T20:15:32.538 INFO:teuthology.orchestra.run.trial098.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.52097.log: -- replaced with /var/log/ceph/cluster2-client.mirror.44962.log.gz 2026-02-22T20:15:32.538 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.51723.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52578.log 2026-02-22T20:15:32.539 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52097.log.gz 2026-02-22T20:15:32.539 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.26636.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.21661.log 2026-02-22T20:15:32.540 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31598.log 2026-02-22T20:15:32.540 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.26636.log.gz 2026-02-22T20:15:32.540 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.21661.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52578.log.gz 2026-02-22T20:15:32.540 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42336.log 2026-02-22T20:15:32.540 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.21661.log.gz 2026-02-22T20:15:32.540 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.31598.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45776.log 2026-02-22T20:15:32.540 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.31598.log.gz 2026-02-22T20:15:32.541 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.42336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47452.log 2026-02-22T20:15:32.541 INFO:teuthology.orchestra.run.trial098.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.3.51723.log.gz 2026-02-22T20:15:32.541 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45776.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38341.log 2026-02-22T20:15:32.541 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.45776.log.gz 2026-02-22T20:15:32.541 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47452.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34306.log 2026-02-22T20:15:32.541 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42336.log.gz 2026-02-22T20:15:32.542 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.38341.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29018.log 2026-02-22T20:15:32.542 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47452.log.gz 2026-02-22T20:15:32.542 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34306.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49501.log 2026-02-22T20:15:32.542 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.38341.log.gz 2026-02-22T20:15:32.542 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.29018.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.34306.log.gz 2026-02-22T20:15:32.542 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50165.log 2026-02-22T20:15:32.542 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.29018.log.gz 2026-02-22T20:15:32.542 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.49501.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.59360.log 2026-02-22T20:15:32.543 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43884.log 2026-02-22T20:15:32.543 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.50165.log: /var/log/ceph/cluster1-client.mirror.3.59360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34333.log 2026-02-22T20:15:32.543 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50165.log.gz 2026-02-22T20:15:32.543 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43884.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49346.log 2026-02-22T20:15:32.543 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34333.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21438.log 2026-02-22T20:15:32.543 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43884.log.gz 2026-02-22T20:15:32.544 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.49346.log: --verbose -- /var/log/ceph/cluster1-client.mirror.53565.log 2026-02-22T20:15:32.544 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.34333.log.gz 2026-02-22T20:15:32.544 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28462.log 2026-02-22T20:15:32.544 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.21438.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49346.log.gz 2026-02-22T20:15:32.544 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53565.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53565.log.gz 2026-02-22T20:15:32.544 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19266.log 2026-02-22T20:15:32.545 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.21438.log.gz 2026-02-22T20:15:32.545 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.28462.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40116.log 2026-02-22T20:15:32.545 INFO:teuthology.orchestra.run.trial098.stderr: 87.7%/var/log/ceph/cluster2-client.admin.19266.log: -- replaced with /var/log/ceph/cluster2-client.mirror.28462.log.gz 2026-02-22T20:15:32.545 INFO:teuthology.orchestra.run.trial098.stderr: 92.8% 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49501.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.3.59360.log.gz 2026-02-22T20:15:32.545 INFO:teuthology.orchestra.run.trial098.stderr:gzip 2026-02-22T20:15:32.545 INFO:teuthology.orchestra.run.trial098.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29715.log 2026-02-22T20:15:32.545 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19266.log.gz 2026-02-22T20:15:32.545 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.40116.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45694.log 2026-02-22T20:15:32.546 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40116.log.gz 2026-02-22T20:15:32.546 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27689.log 2026-02-22T20:15:32.546 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.29715.log: /var/log/ceph/cluster2-client.mirror.45694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37948.log 2026-02-22T20:15:32.546 INFO:teuthology.orchestra.run.trial098.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45694.log.gz 2026-02-22T20:15:32.546 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.27689.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32117.log 2026-02-22T20:15:32.547 INFO:teuthology.orchestra.run.trial098.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29715.log.gz 2026-02-22T20:15:32.547 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37948.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24018.log 2026-02-22T20:15:32.547 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27689.log.gz 2026-02-22T20:15:32.547 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.37948.log.gz 2026-02-22T20:15:32.547 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32117.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39209.log 2026-02-22T20:15:32.547 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.32117.log.gz 2026-02-22T20:15:32.547 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.24018.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48095.log 2026-02-22T20:15:32.547 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39209.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19819.log 2026-02-22T20:15:32.547 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.admin.24018.log.gz 2026-02-22T20:15:32.548 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.48095.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.39209.log.gz 2026-02-22T20:15:32.548 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36589.log 2026-02-22T20:15:32.548 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.48095.log.gz 2026-02-22T20:15:32.548 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19819.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32036.log 2026-02-22T20:15:32.548 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19819.log.gz 2026-02-22T20:15:32.548 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36589.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25855.log 2026-02-22T20:15:32.549 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.36589.log.gz 2026-02-22T20:15:32.549 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32036.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.25202.log 2026-02-22T20:15:32.549 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.32036.log.gz 2026-02-22T20:15:32.549 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.25855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34675.log 2026-02-22T20:15:32.549 INFO:teuthology.orchestra.run.trial098.stderr: 87.5%/var/log/ceph/cluster2-client.mirror.3.25202.log: -- replaced with /var/log/ceph/cluster1-client.admin.25855.log.gz 2026-02-22T20:15:32.549 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-22T20:15:32.550 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.25202.log.gz 2026-02-22T20:15:32.550 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34675.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19083.log 2026-02-22T20:15:32.550 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40191.log 2026-02-22T20:15:32.550 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.34675.log.gz 2026-02-22T20:15:32.550 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32250.log 2026-02-22T20:15:32.551 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19083.log: /var/log/ceph/cluster2-client.admin.40191.log: 80.5% -- replaced with /var/log/ceph/cluster2-client.admin.19083.log.gz 2026-02-22T20:15:32.551 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47916.log 2026-02-22T20:15:32.551 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.40191.log.gz 2026-02-22T20:15:32.551 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.32250.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47389.log 2026-02-22T20:15:32.551 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47916.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32250.log.gz 2026-02-22T20:15:32.551 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45919.log 2026-02-22T20:15:32.552 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47389.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22207.log 2026-02-22T20:15:32.552 INFO:teuthology.orchestra.run.trial098.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.47916.log.gz 2026-02-22T20:15:32.552 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47389.log.gz 2026-02-22T20:15:32.552 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28204.log 2026-02-22T20:15:32.552 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22207.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27865.log 2026-02-22T20:15:32.553 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.22207.log.gz 2026-02-22T20:15:32.553 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.28204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.21336.log 2026-02-22T20:15:32.553 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50213.log 2026-02-22T20:15:32.553 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.27865.log: 89.3%/var/log/ceph/cluster2-client.admin.21336.log: -- replaced with /var/log/ceph/cluster2-client.mirror.28204.log.gz 2026-02-22T20:15:32.553 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44008.log 2026-02-22T20:15:32.553 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.27865.log.gz 2026-02-22T20:15:32.554 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.21336.log.gz 2026-02-22T20:15:32.554 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.50213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33291.log 2026-02-22T20:15:32.554 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44008.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50213.log.gz 2026-02-22T20:15:32.554 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45121.log 2026-02-22T20:15:32.554 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.33291.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23397.log 2026-02-22T20:15:32.554 INFO:teuthology.orchestra.run.trial098.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44008.log.gz 2026-02-22T20:15:32.555 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45121.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30004.log 2026-02-22T20:15:32.555 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33291.log.gz 2026-02-22T20:15:32.555 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.45121.log.gz 2026-02-22T20:15:32.555 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57856.log 2026-02-22T20:15:32.555 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23397.log: /var/log/ceph/cluster1-client.admin.30004.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25777.log 2026-02-22T20:15:32.555 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.30004.log.gz 2026-02-22T20:15:32.555 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.23397.log.gz 2026-02-22T20:15:32.555 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster1-client.mirror.57856.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43904.log 2026-02-22T20:15:32.556 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.25777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24607.log 2026-02-22T20:15:32.556 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57856.log.gz 2026-02-22T20:15:32.556 INFO:teuthology.orchestra.run.trial098.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45919.log.gz 2026-02-22T20:15:32.556 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.25777.log.gz 2026-02-22T20:15:32.556 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34702.log 2026-02-22T20:15:32.556 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24607.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36643.log 2026-02-22T20:15:32.557 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37425.log 2026-02-22T20:15:32.557 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34702.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.24607.log.gz 2026-02-22T20:15:32.557 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19594.log 2026-02-22T20:15:32.557 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.34702.log.gz 2026-02-22T20:15:32.557 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43904.log.gz 2026-02-22T20:15:32.557 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.36643.log.gz 2026-02-22T20:15:32.557 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.37425.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29689.log 2026-02-22T20:15:32.558 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19594.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40862.log 2026-02-22T20:15:32.558 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19594.log.gz 2026-02-22T20:15:32.558 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26901.log 2026-02-22T20:15:32.558 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.29689.log: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.37425.log.gz 2026-02-22T20:15:32.559 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.40862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22321.log 2026-02-22T20:15:32.559 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40862.log.gz 2026-02-22T20:15:32.559 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.29689.log.gz 2026-02-22T20:15:32.559 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.26901.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23991.log 2026-02-22T20:15:32.559 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.26901.log.gz 2026-02-22T20:15:32.559 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.22321.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21582.log 2026-02-22T20:15:32.559 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45322.log 2026-02-22T20:15:32.560 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23991.log: /var/log/ceph/cluster2-client.mirror.21582.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36616.log 2026-02-22T20:15:32.560 INFO:teuthology.orchestra.run.trial098.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.21582.log.gz 2026-02-22T20:15:32.560 INFO:teuthology.orchestra.run.trial098.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.admin.23991.log.gz 2026-02-22T20:15:32.560 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22481.log 2026-02-22T20:15:32.560 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45322.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22321.log.gz 2026-02-22T20:15:32.560 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36616.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29867.log 2026-02-22T20:15:32.560 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45322.log.gz 2026-02-22T20:15:32.560 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.36616.log.gz 2026-02-22T20:15:32.561 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31938.log 2026-02-22T20:15:32.561 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22481.log: /var/log/ceph/cluster1-client.mirror.29867.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19033.log 2026-02-22T20:15:32.561 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.22481.log.gz 2026-02-22T20:15:32.561 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.29867.log.gz 2026-02-22T20:15:32.561 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31938.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31884.log 2026-02-22T20:15:32.561 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.31938.log.gz 2026-02-22T20:15:32.562 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58114.log 2026-02-22T20:15:32.562 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19033.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19033.log.gz 2026-02-22T20:15:32.562 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.31884.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29841.log 2026-02-22T20:15:32.562 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.58114.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.31884.log.gz 2026-02-22T20:15:32.562 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39434.log 2026-02-22T20:15:32.562 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.58114.log.gz 2026-02-22T20:15:32.562 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.29841.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39628.log 2026-02-22T20:15:32.563 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.29841.log.gz 2026-02-22T20:15:32.563 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.39434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38692.log 2026-02-22T20:15:32.563 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26748.log 2026-02-22T20:15:32.563 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.39434.log.gz 2026-02-22T20:15:32.563 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.39628.log: /var/log/ceph/cluster2-client.admin.38692.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26384.log 2026-02-22T20:15:32.563 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.38692.log.gz 2026-02-22T20:15:32.564 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.39628.log.gz 2026-02-22T20:15:32.564 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.26748.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39601.log 2026-02-22T20:15:32.564 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.26384.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.57367.log 2026-02-22T20:15:32.564 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26384.log.gz 2026-02-22T20:15:32.564 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39601.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45799.log 2026-02-22T20:15:32.565 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.57367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23694.log 2026-02-22T20:15:32.565 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.39601.log.gz 2026-02-22T20:15:32.565 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45799.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35266.log 2026-02-22T20:15:32.565 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46830.log 2026-02-22T20:15:32.566 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23694.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.45799.log.gz 2026-02-22T20:15:32.566 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.35266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57952.log 2026-02-22T20:15:32.566 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.23694.log.gz 2026-02-22T20:15:32.566 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37273.log 2026-02-22T20:15:32.566 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35266.log.gz 2026-02-22T20:15:32.566 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.57952.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46442.log 2026-02-22T20:15:32.566 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57952.log.gz 2026-02-22T20:15:32.567 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46830.log.gz 2026-02-22T20:15:32.567 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.37273.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25658.log 2026-02-22T20:15:32.567 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.28356.log 2026-02-22T20:15:32.567 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.37273.log.gz 2026-02-22T20:15:32.567 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.25658.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47660.log 2026-02-22T20:15:32.568 INFO:teuthology.orchestra.run.trial098.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25658.log.gz 2026-02-22T20:15:32.568 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.59253.log 2026-02-22T20:15:32.568 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.28356.log: /var/log/ceph/cluster2-client.mirror.47660.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19058.log 2026-02-22T20:15:32.568 INFO:teuthology.orchestra.run.trial098.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.57367.log.gz 2026-02-22T20:15:32.568 INFO:teuthology.orchestra.run.trial098.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26748.log.gz 2026-02-22T20:15:32.568 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.59253.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47660.log.gzgzip 2026-02-22T20:15:32.568 INFO:teuthology.orchestra.run.trial098.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55900.log 2026-02-22T20:15:32.568 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.28356.log.gz 2026-02-22T20:15:32.569 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49018.log 2026-02-22T20:15:32.569 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19058.log: /var/log/ceph/cluster2-client.mirror.55900.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55900.log.gz 2026-02-22T20:15:32.569 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19058.log.gz 2026-02-22T20:15:32.569 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50048.log 2026-02-22T20:15:32.569 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.49018.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43861.log 2026-02-22T20:15:32.569 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.50048.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.49018.log.gz 2026-02-22T20:15:32.569 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33008.log 2026-02-22T20:15:32.569 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50048.log.gz 2026-02-22T20:15:32.570 INFO:teuthology.orchestra.run.trial098.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46442.log.gz 2026-02-22T20:15:32.570 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32302.log 2026-02-22T20:15:32.570 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.33008.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.33008.log.gz 2026-02-22T20:15:32.570 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22938.log 2026-02-22T20:15:32.570 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39655.log 2026-02-22T20:15:32.570 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32302.log: /var/log/ceph/cluster1-client.admin.22938.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38746.log 2026-02-22T20:15:32.570 INFO:teuthology.orchestra.run.trial098.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43861.log.gz 2026-02-22T20:15:32.571 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.22938.log.gz 2026-02-22T20:15:32.571 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.39655.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.32302.log.gz 2026-02-22T20:15:32.571 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36386.log 2026-02-22T20:15:32.571 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.39655.log.gz 2026-02-22T20:15:32.571 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.38746.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29635.log 2026-02-22T20:15:32.571 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.38746.log.gz 2026-02-22T20:15:32.572 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.36386.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20492.log 2026-02-22T20:15:32.572 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.36386.log.gz 2026-02-22T20:15:32.572 INFO:teuthology.orchestra.run.trial098.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.59253.log.gz 2026-02-22T20:15:32.572 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27130.log 2026-02-22T20:15:32.572 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.29635.log: /var/log/ceph/cluster2-client.mirror.20492.log: 59.0% -- replaced with /var/log/ceph/cluster2-client.mirror.20492.log.gz 2026-02-22T20:15:32.572 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55370.log 2026-02-22T20:15:32.572 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.29635.log.gz 2026-02-22T20:15:32.572 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53453.log 2026-02-22T20:15:32.573 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27130.log: /var/log/ceph/cluster2-client.mirror.55370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43928.log 2026-02-22T20:15:32.573 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53453.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.27130.log.gz 2026-02-22T20:15:32.573 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53453.log.gz 2026-02-22T20:15:32.573 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40316.log 2026-02-22T20:15:32.573 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43928.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.51635.log 2026-02-22T20:15:32.573 INFO:teuthology.orchestra.run.trial098.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.55370.log.gz 2026-02-22T20:15:32.574 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.40316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32640.log 2026-02-22T20:15:32.574 INFO:teuthology.orchestra.run.trial098.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.43928.log.gz 2026-02-22T20:15:32.574 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.51635.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.40316.log.gz 2026-02-22T20:15:32.574 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33774.log 2026-02-22T20:15:32.574 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45591.log 2026-02-22T20:15:32.575 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33774.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.32640.log.gz 2026-02-22T20:15:32.575 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51043.log 2026-02-22T20:15:32.575 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.33774.log.gz 2026-02-22T20:15:32.575 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.45591.log: --verbose -- /var/log/ceph/cluster1-client.mirror.2.59286.log 2026-02-22T20:15:32.575 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.51043.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41511.log 2026-02-22T20:15:32.576 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51043.log.gz 2026-02-22T20:15:32.576 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40343.log 2026-02-22T20:15:32.576 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.59286.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45591.log.gz 2026-02-22T20:15:32.576 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.59286.log.gz 2026-02-22T20:15:32.576 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.41511.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58849.log 2026-02-22T20:15:32.576 INFO:teuthology.orchestra.run.trial098.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.51635.log.gz 2026-02-22T20:15:32.576 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.40343.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33238.log 2026-02-22T20:15:32.577 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.41511.log.gz 2026-02-22T20:15:32.577 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.40343.log.gz 2026-02-22T20:15:32.577 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36045.log 2026-02-22T20:15:32.577 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.58849.log: /var/log/ceph/cluster2-client.admin.33238.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31777.log 2026-02-22T20:15:32.577 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.33238.log.gz 2026-02-22T20:15:32.577 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.58849.log.gz 2026-02-22T20:15:32.577 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36045.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42099.log 2026-02-22T20:15:32.578 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.31777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28973.log 2026-02-22T20:15:32.578 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.36045.log.gz 2026-02-22T20:15:32.578 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.42099.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22659.log 2026-02-22T20:15:32.579 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31777.log.gz 2026-02-22T20:15:32.579 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.42099.log.gz 2026-02-22T20:15:32.579 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.28973.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59878.log 2026-02-22T20:15:32.579 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22659.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.20713.log 2026-02-22T20:15:32.579 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28973.log.gz 2026-02-22T20:15:32.579 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.59878.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53729.log 2026-02-22T20:15:32.579 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.admin.22659.log.gz 2026-02-22T20:15:32.580 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.20713.log: gzip -5 --verbose -- 18.5% /var/log/ceph/cluster1-client.mirror.44189.log -- replaced with /var/log/ceph/cluster1-client.mirror.2.20713.log.gz 2026-02-22T20:15:32.580 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.580 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53729.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40090.log 2026-02-22T20:15:32.580 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44189.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27455.log 2026-02-22T20:15:32.581 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.40090.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57760.log 2026-02-22T20:15:32.581 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44189.log.gz 2026-02-22T20:15:32.581 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36099.log 2026-02-22T20:15:32.581 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.27455.log: /var/log/ceph/cluster1-client.mirror.57760.log: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53729.log.gz 2026-02-22T20:15:32.581 INFO:teuthology.orchestra.run.trial098.stderr:gzip 87.7% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.57760.log.gz --verbose 2026-02-22T20:15:32.582 INFO:teuthology.orchestra.run.trial098.stderr: -- /var/log/ceph/cluster2-client.admin.37761.log 2026-02-22T20:15:32.582 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40090.log.gz 2026-02-22T20:15:32.582 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.27455.log.gz 2026-02-22T20:15:32.582 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.21802.log 2026-02-22T20:15:32.582 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36099.log: /var/log/ceph/cluster2-client.admin.37761.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.37761.log.gz 2026-02-22T20:15:32.582 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25935.log 2026-02-22T20:15:32.582 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.36099.log.gz 2026-02-22T20:15:32.582 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41329.log 2026-02-22T20:15:32.583 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.21802.log: /var/log/ceph/cluster1-client.mirror.25935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.48303.log 2026-02-22T20:15:32.583 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34105.log 2026-02-22T20:15:32.583 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.41329.log: /var/log/ceph/cluster1-client.mirror.1.48303.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.25935.log.gz 2026-02-22T20:15:32.583 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43988.log 2026-02-22T20:15:32.584 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.34105.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.41329.log.gz 2026-02-22T20:15:32.584 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22405.log 2026-02-22T20:15:32.584 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34105.log.gz 2026-02-22T20:15:32.584 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37118.log 2026-02-22T20:15:32.584 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43988.log: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59878.log.gz 2026-02-22T20:15:32.584 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.22405.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56152.log 2026-02-22T20:15:32.584 INFO:teuthology.orchestra.run.trial098.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43988.log.gz 2026-02-22T20:15:32.585 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.37118.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22405.log.gz 2026-02-22T20:15:32.585 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47048.log 2026-02-22T20:15:32.585 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37118.log.gz 2026-02-22T20:15:32.585 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.56152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42581.log 2026-02-22T20:15:32.585 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56152.log.gz 2026-02-22T20:15:32.585 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20128.log 2026-02-22T20:15:32.586 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.47048.log: /var/log/ceph/cluster2-client.mirror.42581.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40397.log 2026-02-22T20:15:32.586 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47048.log.gz 2026-02-22T20:15:32.586 INFO:teuthology.orchestra.run.trial098.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42581.log.gz 2026-02-22T20:15:32.586 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.20128.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42304.log 2026-02-22T20:15:32.586 INFO:teuthology.orchestra.run.trial098.stderr: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.20128.log.gz 2026-02-22T20:15:32.586 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.40397.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41607.log 2026-02-22T20:15:32.587 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.42304.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.40397.log.gz 2026-02-22T20:15:32.587 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21234.log 2026-02-22T20:15:32.587 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.41607.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20303.log 2026-02-22T20:15:32.587 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42304.log.gz 2026-02-22T20:15:32.587 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41607.log.gz 2026-02-22T20:15:32.588 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.21234.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54794.log 2026-02-22T20:15:32.588 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.21234.log.gz 2026-02-22T20:15:32.588 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.20303.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53141.log 2026-02-22T20:15:32.588 INFO:teuthology.orchestra.run.trial098.stderr: 57.2% -- replaced with /var/log/ceph/cluster1-client.admin.20303.log.gz 2026-02-22T20:15:32.588 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.54794.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25991.log 2026-02-22T20:15:32.588 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54794.log.gz 2026-02-22T20:15:32.589 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42197.log 2026-02-22T20:15:32.589 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53141.log: /var/log/ceph/cluster1-client.admin.25991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.57478.log 2026-02-22T20:15:32.589 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53141.log.gz 2026-02-22T20:15:32.589 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.25991.log.gz 2026-02-22T20:15:32.589 INFO:teuthology.orchestra.run.trial098.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.48303.log.gz 2026-02-22T20:15:32.589 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.42197.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37555.log 2026-02-22T20:15:32.590 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.42197.log.gz 2026-02-22T20:15:32.590 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.57478.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44603.log 2026-02-22T20:15:32.590 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.37555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50028.log 2026-02-22T20:15:32.591 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44603.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42845.log 2026-02-22T20:15:32.591 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.37555.log.gz 2026-02-22T20:15:32.591 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44603.log.gz 2026-02-22T20:15:32.591 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44028.log 2026-02-22T20:15:32.591 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.50028.log: /var/log/ceph/cluster2-client.admin.42845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35888.log 2026-02-22T20:15:32.591 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.42845.log.gz 2026-02-22T20:15:32.591 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44028.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.50028.log.gz 2026-02-22T20:15:32.592 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49543.log 2026-02-22T20:15:32.592 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44028.log.gz 2026-02-22T20:15:32.592 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35888.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45397.log 2026-02-22T20:15:32.592 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48964.log 2026-02-22T20:15:32.592 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.35888.log.gz 2026-02-22T20:15:32.592 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.49543.log: /var/log/ceph/cluster1-client.admin.45397.log: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.3.57478.log.gz 2026-02-22T20:15:32.592 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46980.log 2026-02-22T20:15:32.593 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49543.log.gz 2026-02-22T20:15:32.593 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.45397.log.gz 2026-02-22T20:15:32.593 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41862.log 2026-02-22T20:15:32.593 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.48964.log: /var/log/ceph/cluster1-client.mirror.46980.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46980.log.gz 2026-02-22T20:15:32.593 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23558.log 2026-02-22T20:15:32.593 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48964.log.gz 2026-02-22T20:15:32.593 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37734.log 2026-02-22T20:15:32.594 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.41862.log: /var/log/ceph/cluster1-client.admin.23558.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53541.log 2026-02-22T20:15:32.594 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.41862.log.gz 2026-02-22T20:15:32.594 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.23558.log.gz 2026-02-22T20:15:32.594 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.37734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44855.log 2026-02-22T20:15:32.595 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53541.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.37734.log.gz 2026-02-22T20:15:32.595 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44581.log 2026-02-22T20:15:32.595 INFO:teuthology.orchestra.run.trial098.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53541.log.gz 2026-02-22T20:15:32.595 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27665.log 2026-02-22T20:15:32.595 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47305.log 2026-02-22T20:15:32.595 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44581.log: /var/log/ceph/cluster1-client.mirror.27665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35317.log 2026-02-22T20:15:32.596 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44855.log.gz 2026-02-22T20:15:32.596 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.27665.log.gz 2026-02-22T20:15:32.596 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47305.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56251.log 2026-02-22T20:15:32.596 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47305.log.gz 2026-02-22T20:15:32.596 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.35317.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44581.log.gz 2026-02-22T20:15:32.596 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54612.log 2026-02-22T20:15:32.597 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- 87.8% /var/log/ceph/cluster1-client.admin.39057.log -- replaced with /var/log/ceph/cluster1-client.admin.35317.log.gz 2026-02-22T20:15:32.597 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.597 INFO:teuthology.orchestra.run.trial012.stderr: 91.1% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-22T20:15:32.597 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.56251.log: /var/log/ceph/cluster1-client.mirror.54612.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.56251.log.gz 2026-02-22T20:15:32.597 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30058.log 2026-02-22T20:15:32.597 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39057.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54612.log.gz 2026-02-22T20:15:32.597 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29383.log 2026-02-22T20:15:32.597 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.39057.log.gz 2026-02-22T20:15:32.597 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25059.log 2026-02-22T20:15:32.598 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30058.log: /var/log/ceph/cluster1-client.admin.29383.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.30058.log.gz 2026-02-22T20:15:32.598 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19869.log 2026-02-22T20:15:32.598 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.29383.log.gz 2026-02-22T20:15:32.598 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40584.log 2026-02-22T20:15:32.599 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.25059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20228.log 2026-02-22T20:15:32.599 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19869.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.25059.log.gz 2026-02-22T20:15:32.599 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19869.log.gz 2026-02-22T20:15:32.599 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.40584.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53281.log 2026-02-22T20:15:32.599 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.20228.log: gzip 87.5% -5 -- replaced with /var/log/ceph/cluster2-client.admin.40584.log.gz --verbose 2026-02-22T20:15:32.599 INFO:teuthology.orchestra.run.trial098.stderr: -- /var/log/ceph/cluster1-client.mirror.1.54193.log 2026-02-22T20:15:32.599 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.20228.log.gz 2026-02-22T20:15:32.600 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53281.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49448.log 2026-02-22T20:15:32.600 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53281.log.gz 2026-02-22T20:15:32.600 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.54193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35290.log 2026-02-22T20:15:32.600 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.49448.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.49448.log.gz 2026-02-22T20:15:32.600 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42362.log 2026-02-22T20:15:32.600 INFO:teuthology.orchestra.run.trial012.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-22T20:15:32.601 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56200.log 2026-02-22T20:15:32.601 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.35290.log: /var/log/ceph/cluster2-client.mirror.42362.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37166.log 2026-02-22T20:15:32.601 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42362.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.35290.log.gz 2026-02-22T20:15:32.601 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.601 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.56200.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43493.log 2026-02-22T20:15:32.602 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56200.log.gz 2026-02-22T20:15:32.602 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.admin.37166.log: -- /var/log/ceph/cluster1-client.mirror.41971.log 2026-02-22T20:15:32.602 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43493.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43192.log 2026-02-22T20:15:32.602 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.37166.log.gz 2026-02-22T20:15:32.602 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43493.log.gz 2026-02-22T20:15:32.602 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49207.log 2026-02-22T20:15:32.603 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.41971.log: /var/log/ceph/cluster2-client.mirror.43192.log: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43192.log.gz 2026-02-22T20:15:32.603 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21309.log 2026-02-22T20:15:32.603 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41971.log.gz 2026-02-22T20:15:32.603 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22561.log 2026-02-22T20:15:32.603 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.49207.log: /var/log/ceph/cluster1-client.admin.21309.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.21309.log.gz 2026-02-22T20:15:32.603 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.48308.log 2026-02-22T20:15:32.603 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49207.log.gz 2026-02-22T20:15:32.604 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44876.log 2026-02-22T20:15:32.604 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.22561.log: /var/log/ceph/cluster1-client.mirror.2.48308.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.48308.log.gz 2026-02-22T20:15:32.604 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57928.log 2026-02-22T20:15:32.604 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22785.log 2026-02-22T20:15:32.605 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44876.log: /var/log/ceph/cluster1-client.mirror.57928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28567.log 2026-02-22T20:15:32.605 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57928.log.gz 2026-02-22T20:15:32.605 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44876.log.gz 2026-02-22T20:15:32.605 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22785.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31493.log 2026-02-22T20:15:32.605 INFO:teuthology.orchestra.run.trial098.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.22561.log.gz 2026-02-22T20:15:32.605 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28567.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.22785.log.gz 2026-02-22T20:15:32.605 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42564.log 2026-02-22T20:15:32.606 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.31493.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.28567.log.gz 2026-02-22T20:15:32.606 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28818.log 2026-02-22T20:15:32.606 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31493.log.gz 2026-02-22T20:15:32.606 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.42564.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39009.log 2026-02-22T20:15:32.606 INFO:teuthology.orchestra.run.trial098.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42564.log.gz 2026-02-22T20:15:32.606 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46365.log 2026-02-22T20:15:32.607 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.28818.log: /var/log/ceph/cluster1-client.mirror.39009.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39009.log.gz 2026-02-22T20:15:32.607 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53496.log 2026-02-22T20:15:32.607 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28818.log.gz 2026-02-22T20:15:32.607 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46365.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40370.log 2026-02-22T20:15:32.607 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53496.log: gzip 88.0% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.46365.log.gz --verbose 2026-02-22T20:15:32.607 INFO:teuthology.orchestra.run.trial098.stderr: -- /var/log/ceph/cluster2-client.mirror.43249.log 2026-02-22T20:15:32.607 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53496.log.gz 2026-02-22T20:15:32.607 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.40370.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.40370.log.gz 2026-02-22T20:15:32.608 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43145.log 2026-02-22T20:15:32.608 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43249.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54580.log 2026-02-22T20:15:32.608 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43145.log: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.3.21802.log.gz 2026-02-22T20:15:32.608 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44754.log 2026-02-22T20:15:32.609 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29662.log 2026-02-22T20:15:32.609 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.54580.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43145.log.gz 2026-02-22T20:15:32.609 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44754.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54580.log.gz 2026-02-22T20:15:32.609 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25579.log 2026-02-22T20:15:32.609 INFO:teuthology.orchestra.run.trial098.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44754.log.gz 2026-02-22T20:15:32.609 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.29662.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40763.log 2026-02-22T20:15:32.610 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.29662.log.gz 2026-02-22T20:15:32.610 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.25579.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54875.log 2026-02-22T20:15:32.610 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52505.log 2026-02-22T20:15:32.610 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.25579.log.gz 2026-02-22T20:15:32.611 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.40763.log: /var/log/ceph/cluster2-client.mirror.54875.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.40763.log.gz 2026-02-22T20:15:32.611 INFO:teuthology.orchestra.run.trial098.stderr:gzip 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54875.log.gz 2026-02-22T20:15:32.611 INFO:teuthology.orchestra.run.trial098.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.53676.log 2026-02-22T20:15:32.611 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52505.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31624.log 2026-02-22T20:15:32.611 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.53676.log: gzip 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52505.log.gz -5 2026-02-22T20:15:32.611 INFO:teuthology.orchestra.run.trial098.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.42767.log 2026-02-22T20:15:32.612 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.31624.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.53676.log.gz 2026-02-22T20:15:32.612 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44713.log 2026-02-22T20:15:32.612 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58747.log 2026-02-22T20:15:32.612 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.31624.log.gz 2026-02-22T20:15:32.612 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.42767.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43249.log.gz 2026-02-22T20:15:32.612 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44713.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55134.log 2026-02-22T20:15:32.612 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.42767.log.gz 2026-02-22T20:15:32.613 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.58747.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44713.log.gz 2026-02-22T20:15:32.613 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40005.log 2026-02-22T20:15:32.613 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58747.log.gz 2026-02-22T20:15:32.613 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24122.log 2026-02-22T20:15:32.613 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.55134.log: /var/log/ceph/cluster2-client.admin.40005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.55437.log 2026-02-22T20:15:32.614 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.40005.log.gz 2026-02-22T20:15:32.614 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.55134.log.gz 2026-02-22T20:15:32.614 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24122.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44626.log 2026-02-22T20:15:32.614 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.24122.log.gz 2026-02-22T20:15:32.614 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.2.55437.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22234.log 2026-02-22T20:15:32.614 INFO:teuthology.orchestra.run.trial098.stderr: 19.0% -- replaced with /var/log/ceph/cluster2-client.mirror.2.55437.log.gz 2026-02-22T20:15:32.614 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44626.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46167.log 2026-02-22T20:15:32.615 INFO:teuthology.orchestra.run.trial098.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44626.log.gz 2026-02-22T20:15:32.615 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22234.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54751.log 2026-02-22T20:15:32.615 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.admin.22234.log.gz 2026-02-22T20:15:32.616 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19644.log 2026-02-22T20:15:32.616 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26045.log 2026-02-22T20:15:32.616 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.54751.log: /var/log/ceph/cluster2-client.admin.19644.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19644.log.gz 2026-02-22T20:15:32.616 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45940.log 2026-02-22T20:15:32.616 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34575.log 2026-02-22T20:15:32.616 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.26045.log: /var/log/ceph/cluster2-client.mirror.45940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36828.log 2026-02-22T20:15:32.617 INFO:teuthology.orchestra.run.trial098.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45940.log.gz 2026-02-22T20:15:32.617 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.26045.log.gz 2026-02-22T20:15:32.617 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42126.log 2026-02-22T20:15:32.617 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.34575.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54751.log.gz 2026-02-22T20:15:32.617 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36828.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27271.log 2026-02-22T20:15:32.617 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.34575.log.gz 2026-02-22T20:15:32.617 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.36828.log.gz 2026-02-22T20:15:32.618 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.42126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.21670.log 2026-02-22T20:15:32.618 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.42126.log.gz 2026-02-22T20:15:32.618 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.27271.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.59281.log 2026-02-22T20:15:32.618 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.27271.log.gz 2026-02-22T20:15:32.618 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.21670.log: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46167.log.gz 2026-02-22T20:15:32.618 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19494.log 2026-02-22T20:15:32.619 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.59281.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20575.log 2026-02-22T20:15:32.619 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19494.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19494.log.gz 2026-02-22T20:15:32.619 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.21742.log 2026-02-22T20:15:32.619 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.20575.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20575.log.gz 2026-02-22T20:15:32.619 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.20796.log 2026-02-22T20:15:32.620 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.21742.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30580.log 2026-02-22T20:15:32.620 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.20796.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32427.log 2026-02-22T20:15:32.621 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.54296.log 2026-02-22T20:15:32.621 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32427.log: gzip 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.30580.log.gz -5 2026-02-22T20:15:32.621 INFO:teuthology.orchestra.run.trial098.stderr: --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-22T20:15:32.621 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.54296.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.59281.log.gz 2026-02-22T20:15:32.621 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.32427.log.gz 2026-02-22T20:15:32.621 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34387.log 2026-02-22T20:15:32.622 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36197.log 2026-02-22T20:15:32.622 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.34387.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45734.log 2026-02-22T20:15:32.622 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 87.5% --verbose -- -- replaced with /var/log/ceph/cluster2-client.admin.34387.log.gz /var/log/ceph/cluster2-client.mirror.26360.log 2026-02-22T20:15:32.622 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.623 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36197.log: /var/log/ceph/cluster2-client.mirror.45734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31280.log 2026-02-22T20:15:32.623 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.36197.log.gz 2026-02-22T20:15:32.623 INFO:teuthology.orchestra.run.trial098.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.54296.log.gz 2026-02-22T20:15:32.623 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.26360.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.20796.log.gz 2026-02-22T20:15:32.623 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42979.log 2026-02-22T20:15:32.623 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.26360.log.gz 2026-02-22T20:15:32.624 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31280.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31054.log 2026-02-22T20:15:32.624 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.42979.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.31280.log.gz 2026-02-22T20:15:32.624 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55925.log 2026-02-22T20:15:32.624 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.42979.log.gz 2026-02-22T20:15:32.624 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.31054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49045.log 2026-02-22T20:15:32.625 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.55925.log: /var/log/ceph/cluster1-client.mirror.43733.log 2026-02-22T20:15:32.625 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55925.log.gz 2026-02-22T20:15:32.625 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.49045.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31054.log.gz 2026-02-22T20:15:32.625 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22366.log 2026-02-22T20:15:32.625 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43733.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.49045.log.gz 2026-02-22T20:15:32.625 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.20750.log 2026-02-22T20:15:32.625 INFO:teuthology.orchestra.run.trial098.stderr: 90.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43733.log.gz 2026-02-22T20:15:32.626 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.22366.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59941.log 2026-02-22T20:15:32.626 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.20750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29072.log 2026-02-22T20:15:32.626 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.20750.log.gz 2026-02-22T20:15:32.626 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45862.log 2026-02-22T20:15:32.626 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.59941.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45734.log.gz 2026-02-22T20:15:32.627 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.29072.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21396.log 2026-02-22T20:15:32.627 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.29072.log.gz 2026-02-22T20:15:32.627 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45862.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38983.log 2026-02-22T20:15:32.627 INFO:teuthology.orchestra.run.trial098.stderr: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45862.log.gz 2026-02-22T20:15:32.627 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.21396.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29231.log 2026-02-22T20:15:32.628 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.38983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42278.log 2026-02-22T20:15:32.628 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.29231.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33516.log 2026-02-22T20:15:32.629 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38983.log.gz 2026-02-22T20:15:32.629 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.29231.log.gz/var/log/ceph/cluster2-client.admin.42278.log: 2026-02-22T20:15:32.629 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21599.log 2026-02-22T20:15:32.629 INFO:teuthology.orchestra.run.trial098.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21396.log.gz 2026-02-22T20:15:32.629 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.33516.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.42278.log.gz 2026-02-22T20:15:32.629 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46851.log 2026-02-22T20:15:32.629 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.21599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26072.log 2026-02-22T20:15:32.630 INFO:teuthology.orchestra.run.trial098.stderr: 94.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59941.log.gz 2026-02-22T20:15:32.630 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33516.log.gz 2026-02-22T20:15:32.630 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46851.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.21599.log.gz 2026-02-22T20:15:32.630 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20094.log 2026-02-22T20:15:32.630 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46851.log.gz 2026-02-22T20:15:32.630 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.26072.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28046.log 2026-02-22T20:15:32.630 INFO:teuthology.orchestra.run.trial098.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.22366.log.gz 2026-02-22T20:15:32.630 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.20094.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38719.log 2026-02-22T20:15:32.631 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.26072.log.gz 2026-02-22T20:15:32.631 INFO:teuthology.orchestra.run.trial098.stderr: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.20094.log.gz 2026-02-22T20:15:32.631 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.28046.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55633.log 2026-02-22T20:15:32.631 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28046.log.gz 2026-02-22T20:15:32.631 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.38719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19844.log 2026-02-22T20:15:32.632 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.38719.log.gz 2026-02-22T20:15:32.632 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.55633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19442.log 2026-02-22T20:15:32.632 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.55633.log.gz 2026-02-22T20:15:32.632 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58087.log 2026-02-22T20:15:32.632 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19844.log.gz 2026-02-22T20:15:32.632 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51576.log 2026-02-22T20:15:32.632 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19442.log.gz 2026-02-22T20:15:32.633 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.58087.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39316.log 2026-02-22T20:15:32.633 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.51576.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.58087.log.gz 2026-02-22T20:15:32.633 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25035.log 2026-02-22T20:15:32.633 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51576.log.gz 2026-02-22T20:15:32.633 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37662.log 2026-02-22T20:15:32.634 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.39316.log: /var/log/ceph/cluster1-client.mirror.25035.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45006.log 2026-02-22T20:15:32.634 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.25035.log.gz 2026-02-22T20:15:32.634 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.37662.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45714.log 2026-02-22T20:15:32.634 INFO:teuthology.orchestra.run.trial098.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.39316.log.gz 2026-02-22T20:15:32.634 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45006.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45006.log.gz 2026-02-22T20:15:32.634 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31334.log 2026-02-22T20:15:32.635 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.37662.log.gz 2026-02-22T20:15:32.635 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43359.log 2026-02-22T20:15:32.635 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45714.log: /var/log/ceph/cluster2-client.admin.31334.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56331.log 2026-02-22T20:15:32.635 INFO:teuthology.orchestra.run.trial098.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45714.log.gz 2026-02-22T20:15:32.635 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.31334.log.gz 2026-02-22T20:15:32.635 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47824.log 2026-02-22T20:15:32.636 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.56331.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41762.log 2026-02-22T20:15:32.636 INFO:teuthology.orchestra.run.trial098.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43359.log.gz 2026-02-22T20:15:32.636 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47824.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.48347.log 2026-02-22T20:15:32.636 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47824.log.gz 2026-02-22T20:15:32.637 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.41762.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45842.log 2026-02-22T20:15:32.637 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.48347.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49522.log 2026-02-22T20:15:32.637 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41762.log.gz 2026-02-22T20:15:32.637 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45842.log: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.21742.log.gz 2026-02-22T20:15:32.637 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41808.log 2026-02-22T20:15:32.638 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.49522.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45842.log.gz 2026-02-22T20:15:32.638 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53007.log 2026-02-22T20:15:32.638 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41808.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52481.log 2026-02-22T20:15:32.638 INFO:teuthology.orchestra.run.trial098.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49522.log.gz 2026-02-22T20:15:32.638 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53007.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.41808.log.gz 2026-02-22T20:15:32.638 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56278.log 2026-02-22T20:15:32.639 INFO:teuthology.orchestra.run.trial098.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53007.log.gz 2026-02-22T20:15:32.639 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52481.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36072.log 2026-02-22T20:15:32.639 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.56278.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49810.log 2026-02-22T20:15:32.639 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52481.log.gz 2026-02-22T20:15:32.639 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36072.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.36072.log.gz 2026-02-22T20:15:32.639 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42872.log 2026-02-22T20:15:32.639 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.56278.log.gz 2026-02-22T20:15:32.640 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.49810.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19976.log 2026-02-22T20:15:32.640 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.42872.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.49810.log.gz 2026-02-22T20:15:32.640 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41017.log 2026-02-22T20:15:32.640 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.19976.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50291.log 2026-02-22T20:15:32.641 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19976.log.gz 2026-02-22T20:15:32.641 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.admin.42872.log.gz 2026-02-22T20:15:32.641 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.41017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26716.log 2026-02-22T20:15:32.641 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.50291.log: gzip -5 --verbose 88.1% -- -- replaced with /var/log/ceph/cluster2-client.admin.50291.log.gz /var/log/ceph/cluster2-client.mirror.58570.log 2026-02-22T20:15:32.641 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.641 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41017.log.gz 2026-02-22T20:15:32.641 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.26716.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32592.log 2026-02-22T20:15:32.642 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.58570.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58570.log.gz 2026-02-22T20:15:32.642 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37507.log 2026-02-22T20:15:32.642 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.26716.log.gz 2026-02-22T20:15:32.642 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.32592.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44336.log 2026-02-22T20:15:32.643 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.37507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25828.log 2026-02-22T20:15:32.643 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.32592.log.gz 2026-02-22T20:15:32.643 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39461.log 2026-02-22T20:15:32.643 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37507.log.gz 2026-02-22T20:15:32.643 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.25828.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.25828.log.gz 2026-02-22T20:15:32.643 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32200.log 2026-02-22T20:15:32.643 INFO:teuthology.orchestra.run.trial098.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44336.log.gz 2026-02-22T20:15:32.644 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30397.log 2026-02-22T20:15:32.644 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.39461.log: /var/log/ceph/cluster1-client.mirror.32200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30238.log 2026-02-22T20:15:32.644 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.39461.log.gz 2026-02-22T20:15:32.644 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30397.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20269.log 2026-02-22T20:15:32.644 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.32200.log.gz 2026-02-22T20:15:32.645 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30238.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52021.log 2026-02-22T20:15:32.645 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.30397.log.gz 2026-02-22T20:15:32.645 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.20269.log: 55.2% -- replaced with /var/log/ceph/cluster1-client.mirror.20269.log.gz 2026-02-22T20:15:32.645 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.30238.log.gz 2026-02-22T20:15:32.645 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30553.log 2026-02-22T20:15:32.645 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52021.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27429.log 2026-02-22T20:15:32.646 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30553.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52021.log.gz 2026-02-22T20:15:32.646 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33385.log 2026-02-22T20:15:32.646 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.30553.log.gz 2026-02-22T20:15:32.646 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.27429.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32508.log 2026-02-22T20:15:32.646 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.33385.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28121.log 2026-02-22T20:15:32.647 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32508.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.33385.log.gz 2026-02-22T20:15:32.647 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33801.log 2026-02-22T20:15:32.647 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.32508.log.gz 2026-02-22T20:15:32.647 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28121.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.54198.log 2026-02-22T20:15:32.647 INFO:teuthology.orchestra.run.trial098.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.48347.log.gz 2026-02-22T20:15:32.647 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.28121.log.gz/var/log/ceph/cluster1-client.admin.33801.log: 2026-02-22T20:15:32.648 INFO:teuthology.orchestra.run.trial098.stderr: 91.3%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.27429.log.gz -5 2026-02-22T20:15:32.648 INFO:teuthology.orchestra.run.trial098.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.37060.log 2026-02-22T20:15:32.648 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.33801.log.gz 2026-02-22T20:15:32.648 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.54198.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33668.log 2026-02-22T20:15:32.648 INFO:teuthology.orchestra.run.trial098.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.54198.log.gz 2026-02-22T20:15:32.648 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.37060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53967.log 2026-02-22T20:15:32.649 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.33668.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55252.log 2026-02-22T20:15:32.649 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37060.log.gz 2026-02-22T20:15:32.649 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53967.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.33668.log.gz 2026-02-22T20:15:32.649 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35682.log 2026-02-22T20:15:32.649 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.55252.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27554.log 2026-02-22T20:15:32.650 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.55252.log.gz 2026-02-22T20:15:32.650 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35682.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53183.log 2026-02-22T20:15:32.650 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27554.log: gzip -5 87.8% --verbose -- -- replaced with /var/log/ceph/cluster2-client.admin.35682.log.gz /var/log/ceph/cluster1-client.admin.20043.log 2026-02-22T20:15:32.650 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.650 INFO:teuthology.orchestra.run.trial098.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53967.log.gz 2026-02-22T20:15:32.650 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53183.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.27554.log.gz 2026-02-22T20:15:32.650 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36437.log 2026-02-22T20:15:32.651 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.20043.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33726.log 2026-02-22T20:15:32.651 INFO:teuthology.orchestra.run.trial098.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53183.log.gz 2026-02-22T20:15:32.651 INFO:teuthology.orchestra.run.trial098.stderr: 80.6% -- replaced with /var/log/ceph/cluster1-client.admin.20043.log.gz 2026-02-22T20:15:32.651 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38028.log 2026-02-22T20:15:32.652 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.36437.log.gz 2026-02-22T20:15:32.652 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.33726.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45528.log 2026-02-22T20:15:32.652 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.33726.log.gz 2026-02-22T20:15:32.652 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.38028.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41201.log 2026-02-22T20:15:32.652 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45528.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47347.log 2026-02-22T20:15:32.652 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38028.log.gz 2026-02-22T20:15:32.653 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.41201.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39138.log 2026-02-22T20:15:32.653 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47347.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32356.log 2026-02-22T20:15:32.653 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41201.log.gz 2026-02-22T20:15:32.653 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47347.log.gz 2026-02-22T20:15:32.654 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.39138.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42621.log 2026-02-22T20:15:32.654 INFO:teuthology.orchestra.run.trial098.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45528.log.gz 2026-02-22T20:15:32.654 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32356.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.39138.log.gz 2026-02-22T20:15:32.654 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43331.log 2026-02-22T20:15:32.654 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.32356.log.gz 2026-02-22T20:15:32.654 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.42621.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24533.log 2026-02-22T20:15:32.655 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43331.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43080.log 2026-02-22T20:15:32.655 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24533.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50139.log 2026-02-22T20:15:32.655 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57176.log 2026-02-22T20:15:32.656 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43080.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.24533.log.gz 2026-02-22T20:15:32.656 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.50139.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44940.log 2026-02-22T20:15:32.656 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.50139.log.gz 2026-02-22T20:15:32.656 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.57176.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.54165.log 2026-02-22T20:15:32.656 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31517.log 2026-02-22T20:15:32.656 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44940.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57176.log.gz 2026-02-22T20:15:32.657 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.54165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30370.log 2026-02-22T20:15:32.657 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44940.log.gz 2026-02-22T20:15:32.657 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58147.log 2026-02-22T20:15:32.657 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.31517.log.gz 2026-02-22T20:15:32.658 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33642.log 2026-02-22T20:15:32.658 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.30370.log.gz 2026-02-22T20:15:32.658 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.58147.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19769.log 2026-02-22T20:15:32.658 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33642.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31235.log 2026-02-22T20:15:32.659 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.19769.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19769.log.gz 2026-02-22T20:15:32.659 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39515.log 2026-02-22T20:15:32.659 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.33642.log.gz 2026-02-22T20:15:32.659 INFO:teuthology.orchestra.run.trial098.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43331.log.gz 2026-02-22T20:15:32.659 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.31235.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50189.log 2026-02-22T20:15:32.659 INFO:teuthology.orchestra.run.trial098.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42621.log.gz 2026-02-22T20:15:32.659 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.31235.log.gz 2026-02-22T20:15:32.659 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37318.log 2026-02-22T20:15:32.659 INFO:teuthology.orchestra.run.trial098.stderr: 96.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.54193.log.gz 2026-02-22T20:15:32.660 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39515.log: /var/log/ceph/cluster1-client.mirror.50189.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31387.log 2026-02-22T20:15:32.660 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.39515.log.gz 2026-02-22T20:15:32.660 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.37318.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50189.log.gz 2026-02-22T20:15:32.660 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26663.log 2026-02-22T20:15:32.660 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.31387.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.37318.log.gz 2026-02-22T20:15:32.660 INFO:teuthology.orchestra.run.trial098.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43080.log.gz 2026-02-22T20:15:32.660 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58060.log 2026-02-22T20:15:32.660 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.31387.log.gz 2026-02-22T20:15:32.661 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.26663.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38368.log 2026-02-22T20:15:32.661 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.26663.log.gz 2026-02-22T20:15:32.661 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.58060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47578.log 2026-02-22T20:15:32.661 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.58060.log.gz 2026-02-22T20:15:32.662 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.38368.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40836.log 2026-02-22T20:15:32.662 INFO:teuthology.orchestra.run.trial098.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.38368.log.gz 2026-02-22T20:15:32.662 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33615.log 2026-02-22T20:15:32.662 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.40836.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47578.log.gz 2026-02-22T20:15:32.662 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41454.log 2026-02-22T20:15:32.663 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33615.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51555.log 2026-02-22T20:15:32.663 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.33615.log.gz 2026-02-22T20:15:32.663 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41454.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40836.log.gz 2026-02-22T20:15:32.663 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43645.log 2026-02-22T20:15:32.663 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.41454.log.gz 2026-02-22T20:15:32.664 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.51555.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52626.log 2026-02-22T20:15:32.664 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43645.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51555.log.gz 2026-02-22T20:15:32.664 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29285.log 2026-02-22T20:15:32.664 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.52626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32454.log 2026-02-22T20:15:32.665 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.52626.log.gz 2026-02-22T20:15:32.665 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.29285.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49837.log 2026-02-22T20:15:32.665 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.29285.log.gz 2026-02-22T20:15:32.665 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32454.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39084.log 2026-02-22T20:15:32.665 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.43645.log.gz 2026-02-22T20:15:32.666 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.49837.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24495.log 2026-02-22T20:15:32.666 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.32454.log.gz 2026-02-22T20:15:32.666 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39084.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29956.log 2026-02-22T20:15:32.666 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.39084.log.gz 2026-02-22T20:15:32.666 INFO:teuthology.orchestra.run.trial098.stderr: 90.7% -- replaced with /var/log/ceph/cluster1-client.admin.49837.log.gz 2026-02-22T20:15:32.666 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.24495.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42072.log 2026-02-22T20:15:32.666 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.24495.log.gz 2026-02-22T20:15:32.667 INFO:teuthology.orchestra.run.trial098.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.54165.log.gz 2026-02-22T20:15:32.667 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.29956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38844.log 2026-02-22T20:15:32.667 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.29956.log.gz 2026-02-22T20:15:32.667 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.42072.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34077.log 2026-02-22T20:15:32.667 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.42072.log.gz 2026-02-22T20:15:32.668 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.38844.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40736.log 2026-02-22T20:15:32.668 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.38844.log.gz 2026-02-22T20:15:32.668 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.34077.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33059.log 2026-02-22T20:15:32.668 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.40736.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47598.log 2026-02-22T20:15:32.669 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.40736.log.gz 2026-02-22T20:15:32.669 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33059.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34077.log.gz 2026-02-22T20:15:32.669 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23091.log 2026-02-22T20:15:32.669 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.33059.log.gz 2026-02-22T20:15:32.669 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47598.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26098.log 2026-02-22T20:15:32.669 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23091.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44834.log 2026-02-22T20:15:32.670 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.47598.log.gz 2026-02-22T20:15:32.670 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.26098.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.23091.log.gz 2026-02-22T20:15:32.670 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52117.log 2026-02-22T20:15:32.670 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44834.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.26098.log.gz 2026-02-22T20:15:32.670 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36777.log 2026-02-22T20:15:32.670 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44834.log.gz 2026-02-22T20:15:32.671 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52117.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58902.log 2026-02-22T20:15:32.671 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52117.log.gz 2026-02-22T20:15:32.671 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.36777.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39164.log 2026-02-22T20:15:32.671 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36777.log.gz 2026-02-22T20:15:32.671 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.58902.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20286.log 2026-02-22T20:15:32.672 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.39164.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29157.log 2026-02-22T20:15:32.672 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.20286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21619.log 2026-02-22T20:15:32.672 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% 61.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39164.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.20286.log.gz 2026-02-22T20:15:32.672 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.672 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.29157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19519.log 2026-02-22T20:15:32.673 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.21619.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54718.log 2026-02-22T20:15:32.673 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.21619.log.gz 2026-02-22T20:15:32.673 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.19519.log: 90.4%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.29157.log.gz -5 2026-02-22T20:15:32.673 INFO:teuthology.orchestra.run.trial098.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.38644.log 2026-02-22T20:15:32.673 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19519.log.gz 2026-02-22T20:15:32.674 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.54718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30996.log 2026-02-22T20:15:32.674 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.38644.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.54718.log.gz 2026-02-22T20:15:32.674 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39236.log 2026-02-22T20:15:32.674 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38644.log.gz 2026-02-22T20:15:32.674 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30996.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27767.log 2026-02-22T20:15:32.674 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.30996.log.gz 2026-02-22T20:15:32.675 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39236.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43103.log 2026-02-22T20:15:32.675 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46768.log 2026-02-22T20:15:32.675 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.39236.log.gz 2026-02-22T20:15:32.675 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.27767.log.gz 2026-02-22T20:15:32.675 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30085.log 2026-02-22T20:15:32.676 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43103.log: /var/log/ceph/cluster2-client.mirror.46768.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46089.log 2026-02-22T20:15:32.676 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43103.log.gz 2026-02-22T20:15:32.676 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46768.log.gz 2026-02-22T20:15:32.676 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30085.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26542.log 2026-02-22T20:15:32.676 INFO:teuthology.orchestra.run.trial098.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58902.log.gz 2026-02-22T20:15:32.676 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46089.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19367.log 2026-02-22T20:15:32.676 INFO:teuthology.orchestra.run.trial098.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.30085.log.gz 2026-02-22T20:15:32.677 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.26542.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46089.log.gz 2026-02-22T20:15:32.677 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49967.log 2026-02-22T20:15:32.677 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.21706.log 2026-02-22T20:15:32.677 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19367.log.gz 2026-02-22T20:15:32.677 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.49967.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58673.log 2026-02-22T20:15:32.677 INFO:teuthology.orchestra.run.trial098.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.26542.log.gz 2026-02-22T20:15:32.678 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.49967.log.gz 2026-02-22T20:15:32.678 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.21706.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26981.log 2026-02-22T20:15:32.678 INFO:teuthology.orchestra.run.trial098.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.21706.log.gz 2026-02-22T20:15:32.678 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.58673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58876.log 2026-02-22T20:15:32.678 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.26981.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48889.log 2026-02-22T20:15:32.679 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58673.log.gz 2026-02-22T20:15:32.679 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.58876.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49146.log 2026-02-22T20:15:32.679 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.58876.log.gz 2026-02-22T20:15:32.679 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.48889.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26609.log 2026-02-22T20:15:32.680 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.49146.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35655.log 2026-02-22T20:15:32.680 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.48889.log.gz 2026-02-22T20:15:32.680 INFO:teuthology.orchestra.run.trial098.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.26981.log.gz 2026-02-22T20:15:32.680 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.26609.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29258.log 2026-02-22T20:15:32.680 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.49146.log.gz 2026-02-22T20:15:32.680 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.26609.log.gz 2026-02-22T20:15:32.680 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35655.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50508.log 2026-02-22T20:15:32.681 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.35655.log.gz 2026-02-22T20:15:32.681 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.29258.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27076.log 2026-02-22T20:15:32.681 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.29258.log.gz 2026-02-22T20:15:32.681 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.50508.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29608.log 2026-02-22T20:15:32.681 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.27076.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50508.log.gz 2026-02-22T20:15:32.682 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31080.log 2026-02-22T20:15:32.682 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.27076.log.gz 2026-02-22T20:15:32.682 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.29608.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41089.log 2026-02-22T20:15:32.682 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.31080.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.29608.log.gz 2026-02-22T20:15:32.682 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35709.log 2026-02-22T20:15:32.683 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31080.log.gz 2026-02-22T20:15:32.683 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41089.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24045.log 2026-02-22T20:15:32.683 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.41089.log.gz 2026-02-22T20:15:32.683 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.35709.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48245.log 2026-02-22T20:15:32.683 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.24045.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.35709.log.gz 2026-02-22T20:15:32.683 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42743.log 2026-02-22T20:15:32.684 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.24045.log.gz 2026-02-22T20:15:32.684 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.48245.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55183.log 2026-02-22T20:15:32.684 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.42743.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48696.log 2026-02-22T20:15:32.684 INFO:teuthology.orchestra.run.trial098.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48245.log.gz 2026-02-22T20:15:32.684 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.42743.log.gz 2026-02-22T20:15:32.685 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.55183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52680.log 2026-02-22T20:15:32.685 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.55183.log.gz 2026-02-22T20:15:32.685 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.48696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19794.log 2026-02-22T20:15:32.685 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48696.log.gz 2026-02-22T20:15:32.685 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.52680.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47326.log 2026-02-22T20:15:32.686 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52680.log.gz 2026-02-22T20:15:32.686 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19794.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.28593.log 2026-02-22T20:15:32.686 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19794.log.gz 2026-02-22T20:15:32.686 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52554.log 2026-02-22T20:15:32.686 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47326.log.gz 2026-02-22T20:15:32.686 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.28593.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41251.log 2026-02-22T20:15:32.687 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54114.log 2026-02-22T20:15:32.687 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52554.log: /var/log/ceph/cluster2-client.mirror.41251.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.28593.log.gz 2026-02-22T20:15:32.687 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52554.log.gz 2026-02-22T20:15:32.687 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41251.log.gz 2026-02-22T20:15:32.687 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37372.log 2026-02-22T20:15:32.687 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.54114.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37531.log 2026-02-22T20:15:32.688 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37372.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54114.log.gz 2026-02-22T20:15:32.688 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.59244.log 2026-02-22T20:15:32.688 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.37372.log.gz 2026-02-22T20:15:32.688 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.37531.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46109.log 2026-02-22T20:15:32.688 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37531.log.gz 2026-02-22T20:15:32.689 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.59244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22839.log 2026-02-22T20:15:32.689 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.59244.log.gz 2026-02-22T20:15:32.689 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52942.log 2026-02-22T20:15:32.689 INFO:teuthology.orchestra.run.trial098.stderr: 80.0% -- replaced with /var/log/ceph/cluster2-client.mirror.46109.log.gz 2026-02-22T20:15:32.689 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39775.log 2026-02-22T20:15:32.690 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.22839.log.gz 2026-02-22T20:15:32.690 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52942.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40423.log 2026-02-22T20:15:32.690 INFO:teuthology.orchestra.run.trial098.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52942.log.gz 2026-02-22T20:15:32.690 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.39775.log: gzip -5 87.7% --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.39775.log.gz -- 2026-02-22T20:15:32.690 INFO:teuthology.orchestra.run.trial098.stderr: /var/log/ceph/cluster2-client.mirror.43841.log 2026-02-22T20:15:32.691 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.40423.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47473.log 2026-02-22T20:15:32.691 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43841.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41631.log 2026-02-22T20:15:32.691 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.40423.log.gz 2026-02-22T20:15:32.691 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47473.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52965.log 2026-02-22T20:15:32.692 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.41631.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40058.log 2026-02-22T20:15:32.692 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47473.log.gz 2026-02-22T20:15:32.692 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52965.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41631.log.gz 2026-02-22T20:15:32.692 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47990.log 2026-02-22T20:15:32.692 INFO:teuthology.orchestra.run.trial098.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52965.log.gz 2026-02-22T20:15:32.692 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.40058.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37582.log 2026-02-22T20:15:32.692 INFO:teuthology.orchestra.run.trial098.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43841.log.gz 2026-02-22T20:15:32.693 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.47990.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60537.log 2026-02-22T20:15:32.693 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40058.log.gz 2026-02-22T20:15:32.693 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47990.log.gz 2026-02-22T20:15:32.693 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.37582.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46543.log 2026-02-22T20:15:32.693 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.37582.log.gz 2026-02-22T20:15:32.693 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60537.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48835.log 2026-02-22T20:15:32.694 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60537.log.gz 2026-02-22T20:15:32.694 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46543.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46002.log 2026-02-22T20:15:32.694 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.48835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20346.log 2026-02-22T20:15:32.695 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46002.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34877.log 2026-02-22T20:15:32.695 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.admin.48835.log.gz 2026-02-22T20:15:32.695 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29814.log 2026-02-22T20:15:32.695 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.20346.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.20346.log.gz 2026-02-22T20:15:32.695 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.46002.log.gz 2026-02-22T20:15:32.695 INFO:teuthology.orchestra.run.trial098.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56331.log.gz 2026-02-22T20:15:32.695 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.34877.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56019.log 2026-02-22T20:15:32.695 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.34877.log.gz 2026-02-22T20:15:32.695 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24961.log 2026-02-22T20:15:32.696 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.29814.log: /var/log/ceph/cluster2-client.mirror.56019.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.29814.log.gz 2026-02-22T20:15:32.696 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46422.log 2026-02-22T20:15:32.696 INFO:teuthology.orchestra.run.trial098.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56019.log.gz 2026-02-22T20:15:32.696 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24961.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43781.log 2026-02-22T20:15:32.697 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46422.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56059.log 2026-02-22T20:15:32.697 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.43781.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.24961.log.gz 2026-02-22T20:15:32.697 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53822.log 2026-02-22T20:15:32.697 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.46422.log.gz 2026-02-22T20:15:32.697 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.56059.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.43781.log.gz 2026-02-22T20:15:32.697 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51233.log 2026-02-22T20:15:32.698 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53822.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57808.log 2026-02-22T20:15:32.698 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53822.log.gz 2026-02-22T20:15:32.698 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.51233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44296.log 2026-02-22T20:15:32.698 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.admin.51233.log.gz 2026-02-22T20:15:32.698 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster1-client.mirror.57808.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44693.log 2026-02-22T20:15:32.698 INFO:teuthology.orchestra.run.trial098.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46543.log.gz 2026-02-22T20:15:32.699 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44296.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52897.log 2026-02-22T20:15:32.699 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57808.log.gz 2026-02-22T20:15:32.699 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44693.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44296.log.gz 2026-02-22T20:15:32.699 INFO:teuthology.orchestra.run.trial098.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44693.log.gz 2026-02-22T20:15:32.699 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.51668.log 2026-02-22T20:15:32.699 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19694.log 2026-02-22T20:15:32.699 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52897.log: /var/log/ceph/cluster1-client.mirror.3.51668.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.51668.log.gz 2026-02-22T20:15:32.700 INFO:teuthology.orchestra.run.trial098.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52897.log.gz 2026-02-22T20:15:32.700 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34180.log 2026-02-22T20:15:32.700 INFO:teuthology.orchestra.run.trial098.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56059.log.gz 2026-02-22T20:15:32.700 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.19694.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48041.log 2026-02-22T20:15:32.700 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19694.log.gz 2026-02-22T20:15:32.700 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.34180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55875.log 2026-02-22T20:15:32.700 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.34180.log.gz 2026-02-22T20:15:32.701 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.48041.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31155.log 2026-02-22T20:15:32.701 INFO:teuthology.orchestra.run.trial098.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.48041.log.gz 2026-02-22T20:15:32.701 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.55875.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55875.log.gz 2026-02-22T20:15:32.701 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38871.log 2026-02-22T20:15:32.701 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.31155.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48720.log 2026-02-22T20:15:32.702 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.38871.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50960.log 2026-02-22T20:15:32.702 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.31155.log.gz 2026-02-22T20:15:32.702 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38871.log.gz 2026-02-22T20:15:32.702 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.48720.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38898.log 2026-02-22T20:15:32.702 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48720.log.gz 2026-02-22T20:15:32.702 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.50960.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.50960.log.gz 2026-02-22T20:15:32.702 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30633.log 2026-02-22T20:15:32.703 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.50529.log 2026-02-22T20:15:32.703 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.38898.log: /var/log/ceph/cluster1-client.mirror.30633.log: gzip 87.7% -5 -- replaced with /var/log/ceph/cluster2-client.admin.38898.log.gz --verbose 2026-02-22T20:15:32.703 INFO:teuthology.orchestra.run.trial098.stderr: -- /var/log/ceph/cluster1-client.mirror.52809.log 2026-02-22T20:15:32.703 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.50529.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.50529.log.gz 2026-02-22T20:15:32.703 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.30633.log.gz 2026-02-22T20:15:32.703 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55091.log 2026-02-22T20:15:32.704 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35496.log 2026-02-22T20:15:32.704 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.55091.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37092.log 2026-02-22T20:15:32.704 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35496.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58147.log.gz 2026-02-22T20:15:32.705 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34756.log 2026-02-22T20:15:32.705 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55091.log.gz 2026-02-22T20:15:32.705 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.37092.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.35496.log.gz 2026-02-22T20:15:32.705 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 92.2% --verbose -- replaced with /var/log/ceph/cluster1-client.mirror.52809.log.gz -- 2026-02-22T20:15:32.705 INFO:teuthology.orchestra.run.trial098.stderr: /var/log/ceph/cluster2-client.mirror.27200.log 2026-02-22T20:15:32.705 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.34756.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52389.log 2026-02-22T20:15:32.706 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37092.log.gz 2026-02-22T20:15:32.706 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.27200.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.34756.log.gz 2026-02-22T20:15:32.706 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41169.log 2026-02-22T20:15:32.706 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52389.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57880.log 2026-02-22T20:15:32.706 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.41169.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33335.log 2026-02-22T20:15:32.706 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41169.log.gz 2026-02-22T20:15:32.707 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.57880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31965.log 2026-02-22T20:15:32.707 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.33335.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57880.log.gz 2026-02-22T20:15:32.707 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54979.log 2026-02-22T20:15:32.707 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33335.log.gz 2026-02-22T20:15:32.708 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31965.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26302.log 2026-02-22T20:15:32.708 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56039.log 2026-02-22T20:15:32.708 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.31965.log.gz 2026-02-22T20:15:32.708 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.54979.log: /var/log/ceph/cluster2-client.mirror.26302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26018.log 2026-02-22T20:15:32.708 INFO:teuthology.orchestra.run.trial098.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.54979.log.gz 2026-02-22T20:15:32.708 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster2-client.mirror.56039.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42540.log 2026-02-22T20:15:32.709 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.26018.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52389.log.gz 2026-02-22T20:15:32.709 INFO:teuthology.orchestra.run.trial098.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27200.log.gz 2026-02-22T20:15:32.709 INFO:teuthology.orchestra.run.trial098.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26302.log.gz 2026-02-22T20:15:32.709 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57784.log 2026-02-22T20:15:32.709 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.26018.log.gz 2026-02-22T20:15:32.709 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56039.log.gz 2026-02-22T20:15:32.709 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.42540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36517.log 2026-02-22T20:15:32.709 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42540.log.gz 2026-02-22T20:15:32.710 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.57784.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37193.log 2026-02-22T20:15:32.710 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57784.log.gz 2026-02-22T20:15:32.710 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.36517.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37897.log 2026-02-22T20:15:32.710 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36517.log.gz 2026-02-22T20:15:32.710 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41915.log 2026-02-22T20:15:32.710 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47263.log 2026-02-22T20:15:32.711 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.37193.log.gz 2026-02-22T20:15:32.711 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.37897.log: /var/log/ceph/cluster1-client.mirror.41915.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20111.log 2026-02-22T20:15:32.711 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37897.log.gz 2026-02-22T20:15:32.711 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41915.log.gz 2026-02-22T20:15:32.711 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47263.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28893.log 2026-02-22T20:15:32.711 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.20111.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47263.log.gz 2026-02-22T20:15:32.712 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31443.log 2026-02-22T20:15:32.712 INFO:teuthology.orchestra.run.trial098.stderr: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.20111.log.gz 2026-02-22T20:15:32.712 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39410.log 2026-02-22T20:15:32.712 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28893.log: /var/log/ceph/cluster1-client.mirror.31443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35008.log 2026-02-22T20:15:32.712 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.28893.log.gz 2026-02-22T20:15:32.712 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.39410.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.31443.log.gz 2026-02-22T20:15:32.712 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.39410.log.gz 2026-02-22T20:15:32.712 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46651.log 2026-02-22T20:15:32.713 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49394.log 2026-02-22T20:15:32.713 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.35008.log: /var/log/ceph/cluster1-client.mirror.46651.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21105.log 2026-02-22T20:15:32.713 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.49394.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46651.log.gz 2026-02-22T20:15:32.713 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.49394.log.gz 2026-02-22T20:15:32.713 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.35008.log.gz 2026-02-22T20:15:32.713 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19468.log 2026-02-22T20:15:32.714 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster2-client.mirror.21105.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.38773.log 2026-02-22T20:15:32.714 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19468.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19468.log.gz 2026-02-22T20:15:32.714 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48922.log 2026-02-22T20:15:32.714 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.21105.log.gz 2026-02-22T20:15:32.714 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster1-client.admin.38773.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46276.log 2026-02-22T20:15:32.715 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.48922.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.38773.log.gz 2026-02-22T20:15:32.715 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.48352.log 2026-02-22T20:15:32.715 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46276.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31104.log 2026-02-22T20:15:32.715 INFO:teuthology.orchestra.run.trial098.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.48922.log.gz 2026-02-22T20:15:32.715 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster1-client.mirror.3.48352.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35216.log 2026-02-22T20:15:32.716 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.31104.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.48352.log.gz 2026-02-22T20:15:32.716 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.31104.log.gz 2026-02-22T20:15:32.716 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46810.log 2026-02-22T20:15:32.716 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.35216.log: --verbose -- /var/log/ceph/cluster2-client.mirror.44167.log 2026-02-22T20:15:32.716 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46810.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.46810.log.gz 2026-02-22T20:15:32.716 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55274.log 2026-02-22T20:15:32.717 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21085.log 2026-02-22T20:15:32.717 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.35216.log.gz 2026-02-22T20:15:32.717 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.55274.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.admin.55274.log.gz 2026-02-22T20:15:32.717 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24753.log 2026-02-22T20:15:32.718 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54926.log 2026-02-22T20:15:32.718 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.21085.log: /var/log/ceph/cluster1-client.mirror.24753.log: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46276.log.gz 2026-02-22T20:15:32.718 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.24753.log.gz 2026-02-22T20:15:32.718 INFO:teuthology.orchestra.run.trial098.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44167.log.gz 2026-02-22T20:15:32.718 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38547.log 2026-02-22T20:15:32.718 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.21085.log.gz 2026-02-22T20:15:32.718 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.54926.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42464.log 2026-02-22T20:15:32.719 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.54926.log.gz 2026-02-22T20:15:32.719 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.38547.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42437.log 2026-02-22T20:15:32.719 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.42464.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.38547.log.gz 2026-02-22T20:15:32.719 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37247.log 2026-02-22T20:15:32.719 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.42464.log.gz 2026-02-22T20:15:32.719 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33561.log 2026-02-22T20:15:32.720 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.42437.log: /var/log/ceph/cluster1-client.admin.37247.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.37247.log.gz 2026-02-22T20:15:32.720 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19569.log 2026-02-22T20:15:32.720 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.42437.log.gz 2026-02-22T20:15:32.720 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33561.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20543.log 2026-02-22T20:15:32.720 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.19569.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.33561.log.gz 2026-02-22T20:15:32.720 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19569.log.gz 2026-02-22T20:15:32.720 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46299.log 2026-02-22T20:15:32.721 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40611.log 2026-02-22T20:15:32.721 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.20543.log: /var/log/ceph/cluster2-client.mirror.46299.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20543.log.gz 2026-02-22T20:15:32.721 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57832.log 2026-02-22T20:15:32.721 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.40611.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.40611.log.gz 2026-02-22T20:15:32.721 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23478.log 2026-02-22T20:15:32.721 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.46299.log.gz 2026-02-22T20:15:32.722 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39033.log 2026-02-22T20:15:32.722 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.57832.log: /var/log/ceph/cluster1-client.admin.23478.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57832.log.gz 2026-02-22T20:15:32.722 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45899.log 2026-02-22T20:15:32.722 INFO:teuthology.orchestra.run.trial098.stderr: 90.9% -- replaced with /var/log/ceph/cluster1-client.admin.23478.log.gz 2026-02-22T20:15:32.722 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44669.log 2026-02-22T20:15:32.723 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.39033.log: /var/log/ceph/cluster2-client.mirror.45899.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39033.log.gz 2026-02-22T20:15:32.723 INFO:teuthology.orchestra.run.trial098.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.45899.log.gz 2026-02-22T20:15:32.723 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21125.log 2026-02-22T20:15:32.723 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44669.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38493.log 2026-02-22T20:15:32.723 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44669.log.gz 2026-02-22T20:15:32.723 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.21125.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46523.log 2026-02-22T20:15:32.724 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.21125.log.gz 2026-02-22T20:15:32.724 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.38493.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35442.log 2026-02-22T20:15:32.724 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46523.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.38493.log.gz 2026-02-22T20:15:32.724 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38466.log 2026-02-22T20:15:32.724 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46523.log.gz 2026-02-22T20:15:32.725 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.35442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43757.log 2026-02-22T20:15:32.725 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.35442.log.gz 2026-02-22T20:15:32.725 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.38466.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58526.log 2026-02-22T20:15:32.725 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.38466.log.gz 2026-02-22T20:15:32.725 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43757.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30031.log 2026-02-22T20:15:32.726 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.58526.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43757.log.gz 2026-02-22T20:15:32.726 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46671.log 2026-02-22T20:15:32.726 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58526.log.gz 2026-02-22T20:15:32.726 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30031.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30424.log 2026-02-22T20:15:32.726 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.30031.log.gz 2026-02-22T20:15:32.726 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22101.log 2026-02-22T20:15:32.727 INFO:teuthology.orchestra.run.trial098.stderr: 2.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46671.log.gz 2026-02-22T20:15:32.727 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30424.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30111.log 2026-02-22T20:15:32.727 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.22101.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.30424.log.gz 2026-02-22T20:15:32.727 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.22101.log.gz 2026-02-22T20:15:32.727 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35080.log 2026-02-22T20:15:32.728 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.30111.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32667.log 2026-02-22T20:15:32.728 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.35080.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.35080.log.gz 2026-02-22T20:15:32.728 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52760.log 2026-02-22T20:15:32.728 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.30111.log.gz 2026-02-22T20:15:32.728 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20441.log 2026-02-22T20:15:32.728 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52760.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.32667.log.gz 2026-02-22T20:15:32.729 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41581.log 2026-02-22T20:15:32.729 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.20441.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.59309.log 2026-02-22T20:15:32.729 INFO:teuthology.orchestra.run.trial098.stderr: 2.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20441.log.gz 2026-02-22T20:15:32.729 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.41581.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.51607.log 2026-02-22T20:15:32.729 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.52760.log.gz 2026-02-22T20:15:32.730 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.59309.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42693.log 90.6% 2026-02-22T20:15:32.730 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.41581.log.gz 2026-02-22T20:15:32.730 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.51607.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53089.log 2026-02-22T20:15:32.730 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.42693.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.21670.log.gz 2026-02-22T20:15:32.730 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41062.log 2026-02-22T20:15:32.731 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53089.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.50594.log 2026-02-22T20:15:32.731 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42693.log.gz 2026-02-22T20:15:32.731 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28920.log 2026-02-22T20:15:32.731 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41062.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53089.log.gz 2026-02-22T20:15:32.731 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.50594.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32958.log 2026-02-22T20:15:32.731 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.41062.log.gz 2026-02-22T20:15:32.732 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.28920.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47242.log 2026-02-22T20:15:32.732 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.32958.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.2.59309.log.gz 2026-02-22T20:15:32.732 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.28920.log.gz 2026-02-22T20:15:32.732 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20509.log 2026-02-22T20:15:32.733 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47242.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.32958.log.gz 2026-02-22T20:15:32.733 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50487.log 2026-02-22T20:15:32.733 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.20509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30264.log 2026-02-22T20:15:32.733 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47242.log.gz 2026-02-22T20:15:32.733 INFO:teuthology.orchestra.run.trial098.stderr: 58.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20509.log.gz 2026-02-22T20:15:32.733 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.50487.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.50594.log.gz 2026-02-22T20:15:32.733 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28866.log 2026-02-22T20:15:32.734 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.50487.log.gz 2026-02-22T20:15:32.734 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.30264.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42719.log 2026-02-22T20:15:32.734 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53305.log 2026-02-22T20:15:32.734 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.28866.log.gz 2026-02-22T20:15:32.734 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.42719.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30264.log.gz 2026-02-22T20:15:32.734 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38925.log 2026-02-22T20:15:32.734 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42719.log.gz 2026-02-22T20:15:32.735 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.53305.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35469.log 2026-02-22T20:15:32.735 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.53305.log.gz 2026-02-22T20:15:32.735 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.51598.log 2026-02-22T20:15:32.735 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.38925.log: /var/log/ceph/cluster1-client.admin.35469.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.35469.log.gz 2026-02-22T20:15:32.736 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.50543.log 2026-02-22T20:15:32.736 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.38925.log.gz 2026-02-22T20:15:32.736 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.0.51598.log: -- /var/log/ceph/cluster2-client.admin.28540.log 2026-02-22T20:15:32.736 INFO:teuthology.orchestra.run.trial098.stderr: 19.6%/var/log/ceph/cluster1-client.mirror.1.50543.log: -- replaced with /var/log/ceph/cluster1-client.mirror.0.51598.log.gz 2026-02-22T20:15:32.736 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.50543.log.gz 2026-02-22T20:15:32.736 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29464.log 2026-02-22T20:15:32.736 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23721.log 2026-02-22T20:15:32.737 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28540.log: /var/log/ceph/cluster2-client.admin.29464.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.29464.log.gz 2026-02-22T20:15:32.737 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23641.log 2026-02-22T20:15:32.737 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.28540.log.gz 2026-02-22T20:15:32.737 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23721.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24450.log 2026-02-22T20:15:32.738 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23641.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37707.log 2026-02-22T20:15:32.738 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.23721.log.gz 2026-02-22T20:15:32.738 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24450.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59852.log 2026-02-22T20:15:32.738 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.23641.log.gz 2026-02-22T20:15:32.738 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.37707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23531.log 2026-02-22T20:15:32.738 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24450.log.gz 2026-02-22T20:15:32.739 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.59852.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.37707.log.gz 2026-02-22T20:15:32.739 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23614.log 2026-02-22T20:15:32.739 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.59852.log.gz 2026-02-22T20:15:32.739 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23531.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31469.log 2026-02-22T20:15:32.739 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.23531.log.gz 2026-02-22T20:15:32.739 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47536.log 2026-02-22T20:15:32.740 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.23614.log.gz 2026-02-22T20:15:32.740 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.31469.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37481.log 2026-02-22T20:15:32.740 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53031.log 2026-02-22T20:15:32.740 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47536.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.31469.log.gz 2026-02-22T20:15:32.740 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.37481.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.50566.log 2026-02-22T20:15:32.741 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47536.log.gz 2026-02-22T20:15:32.741 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37481.log.gz 2026-02-22T20:15:32.741 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53031.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45044.log 2026-02-22T20:15:32.741 INFO:teuthology.orchestra.run.trial098.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53031.log.gz 2026-02-22T20:15:32.741 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.50566.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57511.log 2026-02-22T20:15:32.742 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45044.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42818.log 2026-02-22T20:15:32.742 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.57511.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45044.log.gz 2026-02-22T20:15:32.742 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24653.log 2026-02-22T20:15:32.742 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57511.log.gz 2026-02-22T20:15:32.742 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.42818.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.54170.log 2026-02-22T20:15:32.743 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.24653.log: gzip -5 --verbose 90.5% -- -- replaced with /var/log/ceph/cluster1-client.admin.42818.log.gz /var/log/ceph/cluster1-client.mirror.51309.log 2026-02-22T20:15:32.743 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.743 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.24653.log.gz 2026-02-22T20:15:32.743 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.54170.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55707.log 2026-02-22T20:15:32.743 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.54170.log.gz 2026-02-22T20:15:32.744 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.51309.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28947.log 2026-02-22T20:15:32.744 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51309.log.gz 2026-02-22T20:15:32.744 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.55707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42952.log 2026-02-22T20:15:32.744 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.55707.log.gz 2026-02-22T20:15:32.744 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.28947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33211.log 2026-02-22T20:15:32.744 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.28947.log.gz 2026-02-22T20:15:32.745 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.42952.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50906.log 2026-02-22T20:15:32.745 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33211.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.42952.log.gz 2026-02-22T20:15:32.745 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.25116.log 2026-02-22T20:15:32.746 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.33211.log.gz 2026-02-22T20:15:32.746 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.50566.log.gzgzip 2026-02-22T20:15:32.746 INFO:teuthology.orchestra.run.trial098.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43123.log 2026-02-22T20:15:32.746 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.50906.log: /var/log/ceph/cluster2-client.mirror.0.25116.log: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.25116.log.gz 2026-02-22T20:15:32.746 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16369.log 2026-02-22T20:15:32.746 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.50906.log.gz 2026-02-22T20:15:32.746 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43123.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19544.log 2026-02-22T20:15:32.747 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.16369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42410.log 2026-02-22T20:15:32.747 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16369.log.gz 2026-02-22T20:15:32.747 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.19544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45203.log 2026-02-22T20:15:32.747 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19544.log.gz 2026-02-22T20:15:32.747 INFO:teuthology.orchestra.run.trial098.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43123.log.gz 2026-02-22T20:15:32.747 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49227.log 2026-02-22T20:15:32.748 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.42410.log: /var/log/ceph/cluster2-client.mirror.45203.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23145.log 2026-02-22T20:15:32.748 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.42410.log.gz 2026-02-22T20:15:32.748 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.49227.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-22T20:15:32.749 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23145.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.23145.log.gz 2026-02-22T20:15:32.749 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54814.log 2026-02-22T20:15:32.749 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33855.log 2026-02-22T20:15:32.749 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.54814.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24316.log 2026-02-22T20:15:32.750 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23172.log 2026-02-22T20:15:32.750 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.24316.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36562.log 2026-02-22T20:15:32.750 INFO:teuthology.orchestra.run.trial098.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.33855.log.gz 2026-02-22T20:15:32.750 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.24316.log.gz 2026-02-22T20:15:32.750 INFO:teuthology.orchestra.run.trial098.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45203.log.gz 2026-02-22T20:15:32.750 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23172.log: gzip -5 --verbose -- 90.2% /var/log/ceph/cluster2-client.mirror.29534.log 2026-02-22T20:15:32.750 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.54814.log.gz 2026-02-22T20:15:32.750 INFO:teuthology.orchestra.run.trial098.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49227.log.gz 2026-02-22T20:15:32.751 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36562.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49736.log 2026-02-22T20:15:32.751 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.29534.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.23172.log.gz 2026-02-22T20:15:32.751 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27607.log 2026-02-22T20:15:32.751 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.36562.log.gz 2026-02-22T20:15:32.751 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.49736.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29534.log.gz 2026-02-22T20:15:32.751 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24343.log 2026-02-22T20:15:32.752 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.27607.log: gzip 89.0% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.49736.log.gz 2026-02-22T20:15:32.752 INFO:teuthology.orchestra.run.trial098.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.20458.log 2026-02-22T20:15:32.752 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.24343.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.27607.log.gz 2026-02-22T20:15:32.752 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.24343.log.gz 2026-02-22T20:15:32.752 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25262.log 2026-02-22T20:15:32.752 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.20458.log: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.20458.log.gz 2026-02-22T20:15:32.752 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38799.log 2026-02-22T20:15:32.753 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.25262.log: --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-22T20:15:32.753 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.38799.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42518.log 2026-02-22T20:15:32.753 INFO:teuthology.orchestra.run.trial098.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.25262.log.gz 2026-02-22T20:15:32.754 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44049.log 2026-02-22T20:15:32.754 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38799.log.gz 2026-02-22T20:15:32.754 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.42518.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48744.log 2026-02-22T20:15:32.754 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.42518.log.gz 2026-02-22T20:15:32.754 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54953.log 2026-02-22T20:15:32.755 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.48744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46913.log 2026-02-22T20:15:32.755 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.54953.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44049.log.gz 2026-02-22T20:15:32.755 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.48744.log.gz 2026-02-22T20:15:32.755 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37921.log 2026-02-22T20:15:32.755 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.54953.log.gz 2026-02-22T20:15:32.755 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46913.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49421.log 2026-02-22T20:15:32.756 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46503.log 2026-02-22T20:15:32.756 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37921.log: /var/log/ceph/cluster1-client.admin.49421.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32900.log 2026-02-22T20:15:32.756 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.37921.log.gz 2026-02-22T20:15:32.756 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46503.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46913.log.gz 2026-02-22T20:15:32.756 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 87.8% --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.46503.log.gz -- 2026-02-22T20:15:32.756 INFO:teuthology.orchestra.run.trial098.stderr: /var/log/ceph/cluster2-client.mirror.47844.log 2026-02-22T20:15:32.757 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.49421.log.gz 2026-02-22T20:15:32.757 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43033.log 2026-02-22T20:15:32.757 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.32900.log: /var/log/ceph/cluster2-client.mirror.47844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53591.log 2026-02-22T20:15:32.757 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.43033.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.32900.log.gz 2026-02-22T20:15:32.757 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35549.log 2026-02-22T20:15:32.757 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.admin.43033.log.gz 2026-02-22T20:15:32.758 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29183.log 2026-02-22T20:15:32.758 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53591.log: /var/log/ceph/cluster2-client.mirror.35549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33700.log 2026-02-22T20:15:32.758 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53591.log.gz 2026-02-22T20:15:32.758 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.29183.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35549.log.gz 2026-02-22T20:15:32.758 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43165.log 2026-02-22T20:15:32.758 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.29183.log.gz 2026-02-22T20:15:32.759 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.33700.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42386.log 2026-02-22T20:15:32.759 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.33700.log.gz 2026-02-22T20:15:32.759 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46188.log 2026-02-22T20:15:32.759 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.42386.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36362.log 2026-02-22T20:15:32.760 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46147.log 2026-02-22T20:15:32.760 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.42386.log.gz 2026-02-22T20:15:32.760 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.36362.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46188.log.gz 2026-02-22T20:15:32.760 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48784.log 2026-02-22T20:15:32.760 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43165.log.gz 2026-02-22T20:15:32.760 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36362.log.gz 92.4% 2026-02-22T20:15:32.760 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.0.51607.log.gz 2026-02-22T20:15:32.760 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46147.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27500.log 2026-02-22T20:15:32.760 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.46147.log.gz 2026-02-22T20:15:32.761 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.48784.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48784.log.gz 2026-02-22T20:15:32.761 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50264.log 2026-02-22T20:15:32.761 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.27500.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47619.log 2026-02-22T20:15:32.761 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.27500.log.gz 2026-02-22T20:15:32.761 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.50264.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21639.log 2026-02-22T20:15:32.762 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.47619.log: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47844.log.gz 2026-02-22T20:15:32.762 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55044.log 2026-02-22T20:15:32.762 INFO:teuthology.orchestra.run.trial098.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.47619.log.gz 2026-02-22T20:15:32.762 INFO:teuthology.orchestra.run.trial098.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.50264.log.gz 2026-02-22T20:15:32.762 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.21639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47721.log 2026-02-22T20:15:32.762 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.55044.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40140.log 2026-02-22T20:15:32.763 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.55044.log.gz 2026-02-22T20:15:32.763 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47721.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30969.log 2026-02-22T20:15:32.763 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.40140.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.40140.log.gz 2026-02-22T20:15:32.763 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59092.log 2026-02-22T20:15:32.764 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55974.log 2026-02-22T20:15:32.764 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.59092.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47721.log.gz 2026-02-22T20:15:32.764 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.30969.log.gz 2026-02-22T20:15:32.764 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60511.log 2026-02-22T20:15:32.764 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59092.log.gz 2026-02-22T20:15:32.764 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.55974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37609.log 2026-02-22T20:15:32.764 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55974.log.gz 2026-02-22T20:15:32.765 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60511.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35736.log 2026-02-22T20:15:32.765 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60511.log.gz 2026-02-22T20:15:32.765 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36304.log 2026-02-22T20:15:32.765 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.37609.log.gz 2026-02-22T20:15:32.766 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.35736.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.12637.log 2026-02-22T20:15:32.766 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.36304.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.35736.log.gz 2026-02-22T20:15:32.766 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43419.log 2026-02-22T20:15:32.766 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/ceph.tmp-client.admin.12637.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36304.log.gz 2026-02-22T20:15:32.766 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.12637.log.gz 2026-02-22T20:15:32.766 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46256.log 2026-02-22T20:15:32.766 INFO:teuthology.orchestra.run.trial098.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.21639.log.gz 2026-02-22T20:15:32.767 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36464.log 2026-02-22T20:15:32.767 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46256.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48942.log 2026-02-22T20:15:32.767 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43419.log.gz 2026-02-22T20:15:32.767 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.46256.log.gz 2026-02-22T20:15:32.767 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35762.log 2026-02-22T20:15:32.768 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.48942.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.36464.log.gz 2026-02-22T20:15:32.768 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.50571.log 2026-02-22T20:15:32.768 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.35762.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49987.log 2026-02-22T20:15:32.769 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60433.log 2026-02-22T20:15:32.769 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.50571.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.50571.log.gz 2026-02-22T20:15:32.769 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.49987.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.35762.log.gz 2026-02-22T20:15:32.769 INFO:teuthology.orchestra.run.trial098.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48942.log.gz 2026-02-22T20:15:32.769 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60028.log 2026-02-22T20:15:32.770 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60433.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35631.log 2026-02-22T20:15:32.770 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.49987.log.gz 2026-02-22T20:15:32.770 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60433.log.gz 2026-02-22T20:15:32.770 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54135.log 2026-02-22T20:15:32.770 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60028.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60028.log.gz 2026-02-22T20:15:32.770 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.35631.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33953.log 2026-02-22T20:15:32.770 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.35631.log.gz 2026-02-22T20:15:32.771 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.54135.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59771.log 2026-02-22T20:15:32.771 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33953.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.54135.log.gz 2026-02-22T20:15:32.771 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57155.log 2026-02-22T20:15:32.771 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.33953.log.gz 2026-02-22T20:15:32.771 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.59771.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40455.log 2026-02-22T20:15:32.771 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.59771.log.gz 2026-02-22T20:15:32.771 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.57155.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59223.log 2026-02-22T20:15:32.772 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.40455.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54771.log 2026-02-22T20:15:32.772 INFO:teuthology.orchestra.run.trial098.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57155.log.gz 2026-02-22T20:15:32.772 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45479.log 2026-02-22T20:15:32.772 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.59223.log: /var/log/ceph/cluster1-client.mirror.54771.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38395.log 2026-02-22T20:15:32.772 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.40455.log.gz 2026-02-22T20:15:32.773 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36278.log 2026-02-22T20:15:32.773 INFO:teuthology.orchestra.run.trial098.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54771.log.gz 2026-02-22T20:15:32.773 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.38395.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43968.log 2026-02-22T20:15:32.773 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.45479.log.gz 2026-02-22T20:15:32.774 INFO:teuthology.orchestra.run.trial098.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.38395.log.gz 2026-02-22T20:15:32.774 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36278.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25525.log 2026-02-22T20:15:32.774 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43968.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.36278.log.gz 2026-02-22T20:15:32.774 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45065.log 2026-02-22T20:15:32.775 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.25525.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56176.log 2026-02-22T20:15:32.775 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43968.log.gz 2026-02-22T20:15:32.775 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.25525.log.gz 2026-02-22T20:15:32.775 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45065.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45065.log.gzgzip 2026-02-22T20:15:32.775 INFO:teuthology.orchestra.run.trial098.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.36491.log 2026-02-22T20:15:32.775 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.56176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59071.log 2026-02-22T20:15:32.775 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56176.log.gz 2026-02-22T20:15:32.775 INFO:teuthology.orchestra.run.trial098.stderr: 94.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59223.log.gz 2026-02-22T20:15:32.775 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43311.log 2026-02-22T20:15:32.776 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36491.log: /var/log/ceph/cluster2-client.mirror.59071.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42021.log 2026-02-22T20:15:32.776 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.36491.log.gz 2026-02-22T20:15:32.776 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43311.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59071.log.gz 2026-02-22T20:15:32.776 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19268.log 2026-02-22T20:15:32.777 INFO:teuthology.orchestra.run.trial098.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.43311.log.gz 2026-02-22T20:15:32.777 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20899.log 2026-02-22T20:15:32.777 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.42021.log: /var/log/ceph/cluster2-client.admin.19268.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19268.log.gz 2026-02-22T20:15:32.777 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20025.log 2026-02-22T20:15:32.777 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42021.log.gz 2026-02-22T20:15:32.777 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51179.log 2026-02-22T20:15:32.777 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.20899.log: /var/log/ceph/cluster1-client.admin.20025.log: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.20025.log.gz 2026-02-22T20:15:32.777 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30157.log 2026-02-22T20:15:32.778 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.51179.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29338.log 2026-02-22T20:15:32.778 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.20899.log.gz 2026-02-22T20:15:32.778 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30157.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.51179.log.gz 2026-02-22T20:15:32.778 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.30157.log.gz 2026-02-22T20:15:32.778 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36726.log 2026-02-22T20:15:32.778 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.57330.log 2026-02-22T20:15:32.778 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.29338.log: /var/log/ceph/cluster1-client.mirror.36726.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26334.log 2026-02-22T20:15:32.779 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.29338.log.gz 2026-02-22T20:15:32.779 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.57330.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43006.log 2026-02-22T20:15:32.779 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.57330.log.gz 2026-02-22T20:15:32.779 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.26334.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.36726.log.gz 2026-02-22T20:15:32.779 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.48280.log 2026-02-22T20:15:32.779 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.43006.log: gzip -5 --verbose 88.0% -- -- replaced with /var/log/ceph/cluster2-client.admin.43006.log.gz /var/log/ceph/cluster1-client.admin.33032.log 2026-02-22T20:15:32.779 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.780 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.48280.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36980.log 2026-02-22T20:15:32.780 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26334.log.gz 2026-02-22T20:15:32.780 INFO:teuthology.orchestra.run.trial098.stderr: 20.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.48280.log.gz 2026-02-22T20:15:32.781 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.admin.33032.log: /var/log/ceph/cluster1-client.admin.40964.log 2026-02-22T20:15:32.781 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36980.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.36980.log.gz 2026-02-22T20:15:32.781 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50466.log 2026-02-22T20:15:32.781 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.33032.log.gz 2026-02-22T20:15:32.781 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.40964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44794.log 2026-02-22T20:15:32.781 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.40964.log.gz 2026-02-22T20:15:32.781 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.50466.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23244.log 2026-02-22T20:15:32.782 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20616.log 2026-02-22T20:15:32.782 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44794.log: /var/log/ceph/cluster1-client.admin.23244.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20633.log 2026-02-22T20:15:32.782 INFO:teuthology.orchestra.run.trial098.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50466.log.gz 2026-02-22T20:15:32.782 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.23244.log.gz 2026-02-22T20:15:32.782 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.20616.log: 55.2% -- replaced with /var/log/ceph/cluster2-client.mirror.20616.log.gz 2026-02-22T20:15:32.782 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44794.log.gz 2026-02-22T20:15:32.783 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49944.log 2026-02-22T20:15:32.783 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.20633.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25011.log 2026-02-22T20:15:32.783 INFO:teuthology.orchestra.run.trial098.stderr: 62.2% -- replaced with /var/log/ceph/cluster2-client.admin.20633.log.gz 2026-02-22T20:15:32.783 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.49944.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45142.log 2026-02-22T20:15:32.784 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.25011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36336.log 2026-02-22T20:15:32.784 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.25011.log.gz 2026-02-22T20:15:32.784 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45142.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39488.log 2026-02-22T20:15:32.784 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.36336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58499.log 2026-02-22T20:15:32.784 INFO:teuthology.orchestra.run.trial098.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45142.log.gz 2026-02-22T20:15:32.784 INFO:teuthology.orchestra.run.trial098.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49944.log.gz 2026-02-22T20:15:32.785 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37034.log 2026-02-22T20:15:32.785 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36336.log.gz 2026-02-22T20:15:32.785 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.39488.log.gz 2026-02-22T20:15:32.785 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.58499.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49296.log 2026-02-22T20:15:32.785 INFO:teuthology.orchestra.run.trial098.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.58499.log.gz 2026-02-22T20:15:32.786 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37034.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49072.log 2026-02-22T20:15:32.786 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.49296.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59798.log 2026-02-22T20:15:32.786 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.37034.log.gz 2026-02-22T20:15:32.786 INFO:teuthology.orchestra.run.trial098.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.49296.log.gz 2026-02-22T20:15:32.786 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40506.log 2026-02-22T20:15:32.786 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.49072.log: /var/log/ceph/cluster1-client.admin.59798.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26874.log 2026-02-22T20:15:32.786 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.49072.log.gz 2026-02-22T20:15:32.786 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.40506.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.59798.log.gz 2026-02-22T20:15:32.787 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40789.log 2026-02-22T20:15:32.787 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40506.log.gz 2026-02-22T20:15:32.787 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.26874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49370.log 2026-02-22T20:15:32.787 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.26874.log.gz 2026-02-22T20:15:32.787 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.40789.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39799.log 2026-02-22T20:15:32.788 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.49370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32063.log 2026-02-22T20:15:32.788 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49370.log.gz 2026-02-22T20:15:32.788 INFO:teuthology.orchestra.run.trial098.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40789.log.gz 2026-02-22T20:15:32.788 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39799.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.20680.log 2026-02-22T20:15:32.789 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32063.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.39799.log.gz 2026-02-22T20:15:32.789 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45981.log 2026-02-22T20:15:32.789 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.32063.log.gz 2026-02-22T20:15:32.789 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.20680.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30211.log 2026-02-22T20:15:32.789 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45981.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52212.log 2026-02-22T20:15:32.789 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30211.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27349.log 2026-02-22T20:15:32.790 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.30211.log.gz 2026-02-22T20:15:32.790 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52212.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47200.log 2026-02-22T20:15:32.790 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52212.log.gz 2026-02-22T20:15:32.790 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24828.log 2026-02-22T20:15:32.790 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47200.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52184.log 2026-02-22T20:15:32.791 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.27349.log.gz 2026-02-22T20:15:32.791 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24828.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47200.log.gz 2026-02-22T20:15:32.791 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.24828.log.gz 2026-02-22T20:15:32.791 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46483.log 2026-02-22T20:15:32.791 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38155.log 2026-02-22T20:15:32.791 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52184.log: /var/log/ceph/cluster2-client.mirror.46483.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40032.log 2026-02-22T20:15:32.791 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.52184.log.gz 2026-02-22T20:15:32.791 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.38155.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46483.log.gz 2026-02-22T20:15:32.791 INFO:teuthology.orchestra.run.trial098.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45981.log.gz 2026-02-22T20:15:32.792 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.38155.log.gz 2026-02-22T20:15:32.792 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48671.log 2026-02-22T20:15:32.792 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5/var/log/ceph/cluster2-client.admin.40032.log: --verbose -- /var/log/ceph/cluster2-client.mirror.58923.log 2026-02-22T20:15:32.792 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.48671.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.48671.log.gz 2026-02-22T20:15:32.792 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22812.log 2026-02-22T20:15:32.792 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.40032.log.gz 2026-02-22T20:15:32.792 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.58923.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34521.log 2026-02-22T20:15:32.793 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22812.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58923.log.gz 2026-02-22T20:15:32.793 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49891.log 2026-02-22T20:15:32.793 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34521.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46958.log 2026-02-22T20:15:32.793 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.34521.log.gz 2026-02-22T20:15:32.793 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.22812.log.gz 2026-02-22T20:15:32.794 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.49891.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41382.log 2026-02-22T20:15:32.794 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46958.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23019.log 2026-02-22T20:15:32.794 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.admin.49891.log.gz 2026-02-22T20:15:32.794 INFO:teuthology.orchestra.run.trial098.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46958.log.gz 2026-02-22T20:15:32.794 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.41382.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43382.log 2026-02-22T20:15:32.795 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.41382.log.gz 2026-02-22T20:15:32.795 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37846.log 2026-02-22T20:15:32.795 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43382.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.20680.log.gz 2026-02-22T20:15:32.795 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.23019.log.gz 2026-02-22T20:15:32.795 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32090.log 2026-02-22T20:15:32.795 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43382.log.gz 2026-02-22T20:15:32.796 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.37846.log: /var/log/ceph/cluster1-client.mirror.30870.log 2026-02-22T20:15:32.796 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32090.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.32090.log.gz 2026-02-22T20:15:32.796 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51937.log 2026-02-22T20:15:32.796 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.30870.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37399.log 2026-02-22T20:15:32.796 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37846.log.gz 2026-02-22T20:15:32.797 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.51937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43709.log 2026-02-22T20:15:32.797 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.30870.log.gz 2026-02-22T20:15:32.797 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51937.log.gz 2026-02-22T20:15:32.797 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.20671.log 2026-02-22T20:15:32.797 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.37399.log.gz 2026-02-22T20:15:32.798 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43709.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36953.log 2026-02-22T20:15:32.798 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster1-client.mirror.0.20671.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34853.log 2026-02-22T20:15:32.798 INFO:teuthology.orchestra.run.trial098.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43709.log.gz 2026-02-22T20:15:32.798 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.20671.log.gz 2026-02-22T20:15:32.798 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36953.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45571.log 87.7% 2026-02-22T20:15:32.798 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.36953.log.gz 2026-02-22T20:15:32.799 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.34853.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43541.log 2026-02-22T20:15:32.799 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34853.log.gz 2026-02-22T20:15:32.799 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33980.log 2026-02-22T20:15:32.799 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45571.log: /var/log/ceph/cluster1-client.mirror.43541.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.43541.log.gz 2026-02-22T20:15:32.799 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38074.log 2026-02-22T20:15:32.799 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.45571.log.gz 2026-02-22T20:15:32.799 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52410.log 2026-02-22T20:15:32.800 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33980.log: /var/log/ceph/cluster1-client.admin.38074.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.33980.log.gz 2026-02-22T20:15:32.800 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58367.log 2026-02-22T20:15:32.800 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.38074.log.gz 2026-02-22T20:15:32.800 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52410.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47557.log 2026-02-22T20:15:32.801 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.58367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24226.log 2026-02-22T20:15:32.801 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.58367.log.gz 2026-02-22T20:15:32.801 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34782.log 2026-02-22T20:15:32.801 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.24226.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47557.log.gz 2026-02-22T20:15:32.801 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31991.log 2026-02-22T20:15:32.802 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.34782.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.24226.log.gz 2026-02-22T20:15:32.802 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46789.log 2026-02-22T20:15:32.802 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.31991.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20390.log 2026-02-22T20:15:32.802 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34782.log.gz 2026-02-22T20:15:32.802 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.31991.log.gz 2026-02-22T20:15:32.802 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46789.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30525.log 2026-02-22T20:15:32.803 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.20390.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33588.log 2026-02-22T20:15:32.803 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46789.log.gz 2026-02-22T20:15:32.803 INFO:teuthology.orchestra.run.trial098.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.20390.log.gz 2026-02-22T20:15:32.803 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30525.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52653.log 2026-02-22T20:15:32.803 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.30525.log.gz 2026-02-22T20:15:32.804 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33588.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.33588.log.gz 2026-02-22T20:15:32.804 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35997.log 2026-02-22T20:15:32.804 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.52653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34153.log 2026-02-22T20:15:32.804 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.35997.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.57407.log 2026-02-22T20:15:32.804 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.52653.log.gz 2026-02-22T20:15:32.804 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35997.log.gz 2026-02-22T20:15:32.805 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.34153.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32694.log 2026-02-22T20:15:32.805 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.57407.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.57407.log.gz 2026-02-22T20:15:32.805 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.34153.log.gz 2026-02-22T20:15:32.805 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50998.log 2026-02-22T20:15:32.805 INFO:teuthology.orchestra.run.trial098.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52410.log.gz 2026-02-22T20:15:32.805 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32694.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27322.log 2026-02-22T20:15:32.805 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.32694.log.gz 2026-02-22T20:15:32.806 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39574.log 2026-02-22T20:15:32.806 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.50998.log: /var/log/ceph/cluster1-client.admin.27322.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56305.log 2026-02-22T20:15:32.806 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.27322.log.gz 2026-02-22T20:15:32.806 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.50998.log.gz 2026-02-22T20:15:32.806 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49322.log 2026-02-22T20:15:32.806 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.39574.log.gz 2026-02-22T20:15:32.807 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.56305.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27892.log 2026-02-22T20:15:32.807 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.56305.log.gz 2026-02-22T20:15:32.807 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.49322.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49322.log.gz 2026-02-22T20:15:32.807 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40164.log 2026-02-22T20:15:32.807 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.27892.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44774.log 2026-02-22T20:15:32.808 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.40164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45879.log 2026-02-22T20:15:32.808 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.27892.log.gz 2026-02-22T20:15:32.808 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44774.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19416.log 2026-02-22T20:15:32.808 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.40164.log.gz 2026-02-22T20:15:32.808 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45879.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.44774.log.gz 2026-02-22T20:15:32.808 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29760.log 2026-02-22T20:15:32.809 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45879.log.gz 2026-02-22T20:15:32.809 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53934.log 2026-02-22T20:15:32.809 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19416.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19416.log.gz 2026-02-22T20:15:32.809 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.29760.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27003.log 2026-02-22T20:15:32.809 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.29760.log.gz 2026-02-22T20:15:32.809 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.53934.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21208.log 2026-02-22T20:15:32.809 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.53934.log.gz 2026-02-22T20:15:32.810 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.27003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32143.log 2026-02-22T20:15:32.810 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.21208.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25086.log 2026-02-22T20:15:32.810 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.32143.log: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.27003.log.gz 2026-02-22T20:15:32.810 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.25125.log 2026-02-22T20:15:32.810 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.21208.log.gz 2026-02-22T20:15:32.810 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.32143.log.gz 2026-02-22T20:15:32.811 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.25086.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52268.log 2026-02-22T20:15:32.811 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.0.25125.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.25086.log.gz 2026-02-22T20:15:32.811 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31697.log 2026-02-22T20:15:32.811 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52268.log: gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-22T20:15:32.811 INFO:teuthology.orchestra.run.trial098.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52268.log.gz 2026-02-22T20:15:32.812 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31697.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26769.log 2026-02-22T20:15:32.812 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.31697.log.gz 2026-02-22T20:15:32.812 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55554.log 2026-02-22T20:15:32.812 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.26769.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49126.log 2026-02-22T20:15:32.813 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.55554.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29125.log 2026-02-22T20:15:32.813 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.55554.log.gz 2026-02-22T20:15:32.813 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.49126.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26769.log.gz 2026-02-22T20:15:32.813 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26961.log 2026-02-22T20:15:32.814 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49126.log.gz/var/log/ceph/cluster1-client.mirror.29125.log: 2026-02-22T20:15:32.814 INFO:teuthology.orchestra.run.trial098.stderr: 87.2% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-22T20:15:32.814 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.25153.log 2026-02-22T20:15:32.814 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.26961.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58167.log 2026-02-22T20:15:32.814 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.29125.log.gz 2026-02-22T20:15:32.814 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.1.25153.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31544.log 2026-02-22T20:15:32.815 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.58167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59899.log 2026-02-22T20:15:32.815 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43229.log 2026-02-22T20:15:32.815 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31544.log: /var/log/ceph/cluster2-client.mirror.59899.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.31544.log.gz 2026-02-22T20:15:32.815 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47368.log 2026-02-22T20:15:32.815 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43229.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26961.log.gz 2026-02-22T20:15:32.816 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.28842.log 2026-02-22T20:15:32.816 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47368.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43229.log.gz 2026-02-22T20:15:32.816 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47368.log.gz 2026-02-22T20:15:32.816 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22965.log 2026-02-22T20:15:32.816 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.28842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.21747.log 2026-02-22T20:15:32.816 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.28842.log.gz 2026-02-22T20:15:32.816 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster1-client.admin.22965.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46564.log 2026-02-22T20:15:32.817 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.21747.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21458.log 2026-02-22T20:15:32.817 INFO:teuthology.orchestra.run.trial098.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.21747.log.gz 2026-02-22T20:15:32.817 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46564.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.22965.log.gz 2026-02-22T20:15:32.817 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22433.log 2026-02-22T20:15:32.818 INFO:teuthology.orchestra.run.trial098.stderr: 89.3%/var/log/ceph/cluster1-client.mirror.21458.log: -- replaced with /var/log/ceph/cluster2-client.mirror.46564.log.gz 2026-02-22T20:15:32.818 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20077.log 2026-02-22T20:15:32.818 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.22433.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.21458.log.gz 2026-02-22T20:15:32.818 INFO:teuthology.orchestra.run.trial098.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59899.log.gz 2026-02-22T20:15:32.818 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose 87.8% -- -- replaced with /var/log/ceph/cluster2-client.mirror.22433.log.gz 2026-02-22T20:15:32.818 INFO:teuthology.orchestra.run.trial098.stderr: /var/log/ceph/cluster1-client.mirror.22457.log 2026-02-22T20:15:32.818 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.20077.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35160.log 2026-02-22T20:15:32.818 INFO:teuthology.orchestra.run.trial098.stderr: 56.2% -- replaced with /var/log/ceph/cluster1-client.mirror.20077.log.gz 2026-02-22T20:15:32.819 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.22457.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.22457.log.gz 2026-02-22T20:15:32.819 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.28412.log 2026-02-22T20:15:32.819 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27919.log 2026-02-22T20:15:32.819 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.35160.log: /var/log/ceph/cluster1-client.mirror.28412.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46728.log 2026-02-22T20:15:32.820 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27919.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47025.log 2026-02-22T20:15:32.820 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.35160.log.gz 2026-02-22T20:15:32.820 INFO:teuthology.orchestra.run.trial098.stderr: 90.5%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.28412.log.gz -5 --verbose 2026-02-22T20:15:32.820 INFO:teuthology.orchestra.run.trial098.stderr: -- /var/log/ceph/cluster2-client.admin.34901.log 2026-02-22T20:15:32.820 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.27919.log.gz 2026-02-22T20:15:32.820 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46728.log: /var/log/ceph/cluster1-client.mirror.47025.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47025.log.gz 2026-02-22T20:15:32.820 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40886.log 2026-02-22T20:15:32.820 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.46728.log.gz 2026-02-22T20:15:32.821 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.34901.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.51663.log 2026-02-22T20:15:32.821 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.40886.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.34901.log.gz 2026-02-22T20:15:32.821 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.40886.log.gz 2026-02-22T20:15:32.821 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42152.log 2026-02-22T20:15:32.821 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23892.log 2026-02-22T20:15:32.822 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.51663.log: /var/log/ceph/cluster1-client.mirror.42152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31571.log 2026-02-22T20:15:32.822 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.42152.log.gz 2026-02-22T20:15:32.822 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51020.log/var/log/ceph/cluster1-client.admin.23892.log: 2026-02-22T20:15:32.822 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.31571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39360.log 2026-02-22T20:15:32.822 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.23892.log.gz 2026-02-22T20:15:32.822 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.31571.log.gz 2026-02-22T20:15:32.822 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.51020.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37636.log 2026-02-22T20:15:32.823 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.51020.log.gz 2026-02-22T20:15:32.823 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.39360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22341.log 2026-02-22T20:15:32.823 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37636.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.57344.log 2026-02-22T20:15:32.823 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.37636.log.gz 2026-02-22T20:15:32.823 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39360.log.gz 2026-02-22T20:15:32.823 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.22341.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59202.log 2026-02-22T20:15:32.824 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.57344.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55017.log 2026-02-22T20:15:32.824 INFO:teuthology.orchestra.run.trial098.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.2.51663.log.gz 2026-02-22T20:15:32.824 INFO:teuthology.orchestra.run.trial098.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.57344.log.gz 2026-02-22T20:15:32.824 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42898.log 2026-02-22T20:15:32.824 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.59202.log: /var/log/ceph/cluster1-client.admin.55017.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59983.log 2026-02-22T20:15:32.824 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.55017.log.gz 2026-02-22T20:15:32.825 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.59202.log.gz 2026-02-22T20:15:32.825 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.42898.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35807.log 2026-02-22T20:15:32.825 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.22341.log.gz 2026-02-22T20:15:32.825 INFO:teuthology.orchestra.run.trial098.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42898.log.gz 2026-02-22T20:15:32.825 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.59983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20372.log 2026-02-22T20:15:32.825 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49187.log 2026-02-22T20:15:32.825 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.35807.log: /var/log/ceph/cluster2-client.admin.20372.log: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.20372.log.gz 2026-02-22T20:15:32.826 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41655.log 2026-02-22T20:15:32.826 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.35807.log.gz 2026-02-22T20:15:32.826 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.49187.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41736.log 2026-02-22T20:15:32.826 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49187.log.gz 2026-02-22T20:15:32.826 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.41655.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26462.log 2026-02-22T20:15:32.827 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47763.log 2026-02-22T20:15:32.827 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.41655.log.gz 2026-02-22T20:15:32.827 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41736.log: 94.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59983.log.gz 2026-02-22T20:15:32.827 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.26462.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28789.log 2026-02-22T20:15:32.827 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.26462.log.gz 2026-02-22T20:15:32.827 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.41736.log.gz 2026-02-22T20:15:32.827 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47763.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38573.log 2026-02-22T20:15:32.828 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.28789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52987.log 2026-02-22T20:15:32.828 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.38573.log: /var/log/ceph/cluster2-client.mirror.47431.log 2026-02-22T20:15:32.828 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47763.log.gz 2026-02-22T20:15:32.828 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52987.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28789.log.gz 2026-02-22T20:15:32.829 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52325.log 2026-02-22T20:15:32.829 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52987.log.gz 2026-02-22T20:15:32.829 INFO:teuthology.orchestra.run.trial098.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38573.log.gz 2026-02-22T20:15:32.829 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47431.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30739.log 2026-02-22T20:15:32.829 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52325.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47431.log.gz 2026-02-22T20:15:32.829 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29207.log 2026-02-22T20:15:32.830 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.30739.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36753.log 2026-02-22T20:15:32.830 INFO:teuthology.orchestra.run.trial098.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52325.log.gz 2026-02-22T20:15:32.830 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.29207.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30739.log.gz 2026-02-22T20:15:32.830 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19744.log 2026-02-22T20:15:32.830 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29207.log.gz 2026-02-22T20:15:32.830 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.36753.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29929.log 2026-02-22T20:15:32.831 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36753.log.gz 2026-02-22T20:15:32.831 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.19744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.22288.log 2026-02-22T20:15:32.831 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19744.log.gz 2026-02-22T20:15:32.831 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.29929.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26435.log 2026-02-22T20:15:32.831 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.22288.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26162.log 2026-02-22T20:15:32.832 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.26435.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48862.log 2026-02-22T20:15:32.832 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.29929.log.gz 2026-02-22T20:15:32.832 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.26435.log.gz 2026-02-22T20:15:32.832 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.22288.log.gz 2026-02-22T20:15:32.832 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.26162.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29490.log 2026-02-22T20:15:32.832 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.48862.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45450.log 2026-02-22T20:15:32.833 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.29490.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.48862.log.gz 2026-02-22T20:15:32.833 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24516.log 2026-02-22T20:15:32.833 INFO:teuthology.orchestra.run.trial098.stderr: 90.7% -- replaced with /var/log/ceph/cluster1-client.mirror.26162.log.gz 2026-02-22T20:15:32.833 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45450.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27376.log 2026-02-22T20:15:32.833 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.29490.log.gz 2026-02-22T20:15:32.834 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24516.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29099.log 2026-02-22T20:15:32.834 INFO:teuthology.orchestra.run.trial098.stderr: 66.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24516.log.gz 2026-02-22T20:15:32.834 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26489.log 2026-02-22T20:15:32.834 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.27376.log.gz 2026-02-22T20:15:32.834 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.29099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21183.log 2026-02-22T20:15:32.834 INFO:teuthology.orchestra.run.trial098.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.45450.log.gz 2026-02-22T20:15:32.834 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20060.log 2026-02-22T20:15:32.835 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.26489.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.29099.log.gz 2026-02-22T20:15:32.835 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.21183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53048.log 2026-02-22T20:15:32.835 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.26489.log.gz 2026-02-22T20:15:32.835 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24253.log/var/log/ceph/cluster1-client.admin.20060.log: 2026-02-22T20:15:32.835 INFO:teuthology.orchestra.run.trial098.stderr: 66.0% -- replaced with /var/log/ceph/cluster1-client.admin.20060.log.gz 2026-02-22T20:15:32.836 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30715.log 2026-02-22T20:15:32.836 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.24253.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26823.log 2026-02-22T20:15:32.836 INFO:teuthology.orchestra.run.trial098.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53048.log.gz 2026-02-22T20:15:32.836 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43209.log 2026-02-22T20:15:32.836 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.24253.log.gz 2026-02-22T20:15:32.837 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.30715.log: /var/log/ceph/cluster2-client.mirror.26823.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31128.log 2026-02-22T20:15:32.837 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.26823.log.gz 2026-02-22T20:15:32.837 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30715.log.gz 2026-02-22T20:15:32.837 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43209.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19669.log 2026-02-22T20:15:32.837 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43209.log.gz 2026-02-22T20:15:32.837 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46463.log 2026-02-22T20:15:32.837 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.31128.log: /var/log/ceph/cluster1-client.admin.19669.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19669.log.gz 2026-02-22T20:15:32.837 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53365.log 2026-02-22T20:15:32.838 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.31128.log.gz 2026-02-22T20:15:32.838 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19259.log 2026-02-22T20:15:32.838 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53365.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46463.log.gz 2026-02-22T20:15:32.838 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33881.log 2026-02-22T20:15:32.839 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41709.log 2026-02-22T20:15:32.839 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19259.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19259.log.gz 2026-02-22T20:15:32.839 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.33881.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37142.log 2026-02-22T20:15:32.839 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.33881.log.gz 2026-02-22T20:15:32.839 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41709.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24939.log 2026-02-22T20:15:32.839 INFO:teuthology.orchestra.run.trial098.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.21183.log.gz 2026-02-22T20:15:32.839 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53365.log.gz 2026-02-22T20:15:32.840 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.37142.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.41709.log.gz 2026-02-22T20:15:32.840 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28692.log 2026-02-22T20:15:32.840 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37142.log.gz 2026-02-22T20:15:32.840 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60052.log 2026-02-22T20:15:32.840 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28692.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60485.log 2026-02-22T20:15:32.841 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.60052.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58943.log 2026-02-22T20:15:32.841 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.28692.log.gz 2026-02-22T20:15:32.841 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60052.log.gz 2026-02-22T20:15:32.841 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60485.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60485.log.gz 2026-02-22T20:15:32.841 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33086.log 2026-02-22T20:15:32.842 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.58943.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.55409.log 2026-02-22T20:15:32.842 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33086.log: gzip 87.6% -5 -- replaced with /var/log/ceph/cluster2-client.admin.33086.log.gz --verbose 2026-02-22T20:15:32.842 INFO:teuthology.orchestra.run.trial098.stderr: -- /var/log/ceph/cluster1-client.admin.39978.log 2026-02-22T20:15:32.842 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.1.55409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27156.log 2026-02-22T20:15:32.842 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.1.55409.log.gz 2026-02-22T20:15:32.843 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39978.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32820.log 2026-02-22T20:15:32.843 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.39978.log.gz 2026-02-22T20:15:32.843 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.27156.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.28438.log 2026-02-22T20:15:32.844 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32820.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37873.log 2026-02-22T20:15:32.844 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.27156.log.gz 2026-02-22T20:15:32.844 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.32820.log.gz 2026-02-22T20:15:32.844 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.28438.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53428.log 2026-02-22T20:15:32.844 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.28438.log.gz 2026-02-22T20:15:32.845 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.37873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.50654.log 2026-02-22T20:15:32.845 INFO:teuthology.orchestra.run.trial098.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24939.log.gz 2026-02-22T20:15:32.845 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37873.log.gz 2026-02-22T20:15:32.845 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53428.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52789.log 2026-02-22T20:15:32.845 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.50654.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42251.log 2026-02-22T20:15:32.845 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30184.log 2026-02-22T20:15:32.846 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52789.log: /var/log/ceph/cluster2-client.admin.42251.log: gzip 90.5% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.53428.log.gz --verbose 2026-02-22T20:15:32.846 INFO:teuthology.orchestra.run.trial098.stderr: -- /var/log/ceph/cluster2-client.mirror.32566.log 2026-02-22T20:15:32.846 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.42251.log.gz 2026-02-22T20:15:32.846 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30184.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16452.log 2026-02-22T20:15:32.846 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.30184.log.gz 2026-02-22T20:15:32.847 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.32566.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38314.log 2026-02-22T20:15:32.847 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.16452.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52789.log.gz 2026-02-22T20:15:32.847 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51734.log 2026-02-22T20:15:32.847 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16452.log.gz 2026-02-22T20:15:32.847 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.38314.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32566.log.gz 2026-02-22T20:15:32.847 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38668.log 2026-02-22T20:15:32.847 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.38314.log.gz 2026-02-22T20:15:32.848 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25957.log 2026-02-22T20:15:32.848 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.51734.log: /var/log/ceph/cluster1-client.mirror.38668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25915.log 2026-02-22T20:15:32.848 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51734.log.gz 2026-02-22T20:15:32.848 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.38668.log.gz 2026-02-22T20:15:32.848 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.25957.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.50654.log.gz 2026-02-22T20:15:32.848 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58009.log 2026-02-22T20:15:32.848 INFO:teuthology.orchestra.run.trial098.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.1.25153.log.gz 2026-02-22T20:15:32.848 INFO:teuthology.orchestra.run.trial098.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.25957.log.gz 2026-02-22T20:15:32.849 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.25915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34360.log 2026-02-22T20:15:32.849 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.58009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27972.log 2026-02-22T20:15:32.849 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.34360.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58009.log.gz 2026-02-22T20:15:32.849 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- 87.7% /var/log/ceph/cluster1-client.admin.41427.log -- replaced with /var/log/ceph/cluster2-client.admin.34360.log.gz 2026-02-22T20:15:32.849 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.850 INFO:teuthology.orchestra.run.trial098.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.25915.log.gz 2026-02-22T20:15:32.850 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.27972.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57198.log 2026-02-22T20:15:32.850 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41427.log: gzip 87.7% -5 --verbose -- replaced with /var/log/ceph/cluster1-client.admin.41427.log.gz -- 2026-02-22T20:15:32.850 INFO:teuthology.orchestra.run.trial098.stderr: /var/log/ceph/cluster1-client.admin.43808.log 2026-02-22T20:15:32.850 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.27972.log.gz 2026-02-22T20:15:32.850 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.57198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24369.log 2026-02-22T20:15:32.851 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57198.log.gz 2026-02-22T20:15:32.851 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.43808.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29584.log 2026-02-22T20:15:32.851 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33361.log 2026-02-22T20:15:32.852 INFO:teuthology.orchestra.run.trial098.stderr: 90.9% -- replaced with /var/log/ceph/cluster1-client.admin.43808.log.gz 2026-02-22T20:15:32.852 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.29584.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.24369.log.gz 2026-02-22T20:15:32.852 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19103.log 2026-02-22T20:15:32.852 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.29584.log.gz 2026-02-22T20:15:32.852 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.33361.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53850.log 2026-02-22T20:15:32.852 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33361.log.gz 2026-02-22T20:15:32.853 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19103.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.57339.log 2026-02-22T20:15:32.853 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19103.log.gz 2026-02-22T20:15:32.853 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53850.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33463.log 2026-02-22T20:15:32.853 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53850.log.gz 2026-02-22T20:15:32.854 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.57339.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.48266.log 2026-02-22T20:15:32.854 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25632.log 2026-02-22T20:15:32.854 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.33463.log.gz/var/log/ceph/cluster1-client.mirror.0.48266.log: 2026-02-22T20:15:32.854 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48433.log 2026-02-22T20:15:32.854 INFO:teuthology.orchestra.run.trial098.stderr: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58167.log.gz 2026-02-22T20:15:32.854 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.48266.log.gz 2026-02-22T20:15:32.855 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.25632.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33926.log 2026-02-22T20:15:32.855 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.48433.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.25632.log.gz 2026-02-22T20:15:32.855 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34447.log 87.4% 2026-02-22T20:15:32.855 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.48433.log.gz 2026-02-22T20:15:32.855 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33926.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59825.log 2026-02-22T20:15:32.856 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.33926.log.gz 2026-02-22T20:15:32.856 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.34447.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.59314.log 2026-02-22T20:15:32.856 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.admin.59825.log: -- /var/log/ceph/cluster1-client.admin.28276.log 2026-02-22T20:15:32.856 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.59314.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39541.log 2026-02-22T20:15:32.857 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.34447.log.gz 2026-02-22T20:15:32.857 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.59825.log.gz 2026-02-22T20:15:32.857 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.59314.log.gz 2026-02-22T20:15:32.857 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.28276.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20320.log 2026-02-22T20:15:32.857 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36152.log 2026-02-22T20:15:32.857 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.28276.log.gz 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.39541.log: /var/log/ceph/cluster2-client.admin.20320.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44918.log 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.20320.log.gz 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.36152.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30942.log 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.36152.log.gz 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39541.log.gz 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44918.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24778.log 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30942.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32616.log 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.30942.log.gz 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24778.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52138.log 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.44918.log.gz 2026-02-22T20:15:32.859 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.24778.log.gz 2026-02-22T20:15:32.860 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.32616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45615.log 2026-02-22T20:15:32.860 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.32616.log.gz 2026-02-22T20:15:32.860 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45653.log 2026-02-22T20:15:32.860 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52138.log: /var/log/ceph/cluster2-client.mirror.45615.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52138.log.gz 2026-02-22T20:15:32.860 INFO:teuthology.orchestra.run.trial098.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45615.log.gz 2026-02-22T20:15:32.860 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23838.log 2026-02-22T20:15:32.860 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50445.log 2026-02-22T20:15:32.860 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23838.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45653.log.gz 2026-02-22T20:15:32.861 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.23838.log.gz 2026-02-22T20:15:32.861 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55070.log 2026-02-22T20:15:32.861 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.59258.log 2026-02-22T20:15:32.861 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.50445.log: /var/log/ceph/cluster1-client.mirror.55070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42045.log 2026-02-22T20:15:32.861 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40530.log 2026-02-22T20:15:32.862 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.59258.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55070.log.gz 2026-02-22T20:15:32.862 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.42045.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.59258.log.gz 2026-02-22T20:15:32.862 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60639.log 2026-02-22T20:15:32.862 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.42045.log.gz 2026-02-22T20:15:32.862 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.40530.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34497.log 2026-02-22T20:15:32.862 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.40530.log.gz 2026-02-22T20:15:32.862 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60639.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60639.log.gz 2026-02-22T20:15:32.862 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26522.log 2026-02-22T20:15:32.863 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.34497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37220.log 2026-02-22T20:15:32.863 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.34497.log.gz 2026-02-22T20:15:32.863 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.26522.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54657.log 2026-02-22T20:15:32.863 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.37220.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44561.log 2026-02-22T20:15:32.863 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.37220.log.gz 2026-02-22T20:15:32.864 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.54657.log: gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-22T20:15:32.864 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47157.log 2026-02-22T20:15:32.864 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44561.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.54657.log.gz 2026-02-22T20:15:32.864 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2.audit.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.26522.log.gz 2026-02-22T20:15:32.864 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20424.log 2026-02-22T20:15:32.865 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47157.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44561.log.gz 2026-02-22T20:15:32.865 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37007.log 2026-02-22T20:15:32.865 INFO:teuthology.orchestra.run.trial098.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47157.log.gz 2026-02-22T20:15:32.865 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.20424.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60407.log 2026-02-22T20:15:32.865 INFO:teuthology.orchestra.run.trial098.stderr: 55.8% -- replaced with /var/log/ceph/cluster2-client.mirror.20424.log.gz 2026-02-22T20:15:32.865 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40271.log 2026-02-22T20:15:32.865 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.37007.log.gz 2026-02-22T20:15:32.866 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60407.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60407.log.gz 2026-02-22T20:15:32.866 INFO:teuthology.orchestra.run.trial098.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50445.log.gz 2026-02-22T20:15:32.866 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20145.log 2026-02-22T20:15:32.866 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55210.log 2026-02-22T20:15:32.866 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.40271.log: /var/log/ceph/cluster1-client.mirror.20145.log: 58.3% -- replaced with /var/log/ceph/cluster1-client.mirror.20145.log.gz 2026-02-22T20:15:32.866 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41889.log 2026-02-22T20:15:32.867 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50237.log 2026-02-22T20:15:32.867 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40271.log.gz 2026-02-22T20:15:32.867 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.55210.log: /var/log/ceph/cluster2-client.admin.41889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52345.log 2026-02-22T20:15:32.867 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.41889.log.gz 87.8% 2026-02-22T20:15:32.867 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.55210.log.gz 2026-02-22T20:15:32.867 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.50237.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24703.log 2026-02-22T20:15:32.868 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.50237.log.gz 2026-02-22T20:15:32.868 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31724.log 2026-02-22T20:15:32.868 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.24703.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52345.log.gz 2026-02-22T20:15:32.868 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49564.log 2026-02-22T20:15:32.868 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.24703.log.gz 2026-02-22T20:15:32.869 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.31724.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38128.log 2026-02-22T20:15:32.869 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.31724.log.gz 2026-02-22T20:15:32.869 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.49564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28745.log 2026-02-22T20:15:32.869 INFO:teuthology.orchestra.run.trial098.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.49564.log.gz 2026-02-22T20:15:32.869 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.38128.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28249.log 2026-02-22T20:15:32.870 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.38128.log.gzgzip 2026-02-22T20:15:32.870 INFO:teuthology.orchestra.run.trial098.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44231.log 2026-02-22T20:15:32.870 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.28745.log: /var/log/ceph/cluster1-client.admin.28249.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60381.log 2026-02-22T20:15:32.871 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.28249.log.gz 2026-02-22T20:15:32.871 INFO:teuthology.orchestra.run.trial098.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28745.log.gz 2026-02-22T20:15:32.871 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44231.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32534.log 2026-02-22T20:15:32.871 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60381.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44231.log.gz 2026-02-22T20:15:32.871 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56224.log 2026-02-22T20:15:32.871 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60381.log.gz 2026-02-22T20:15:32.871 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.32534.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58546.log 2026-02-22T20:15:32.871 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.56224.log: 93.8% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-22T20:15:32.872 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39826.log 2026-02-22T20:15:32.872 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32534.log.gz 2026-02-22T20:15:32.872 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.58546.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.56224.log.gz 2026-02-22T20:15:32.872 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25704.log 2026-02-22T20:15:32.872 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39826.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30689.log 2026-02-22T20:15:32.873 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.39826.log.gz 2026-02-22T20:15:32.873 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.25704.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47093.log 2026-02-22T20:15:32.873 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.30689.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54634.log 2026-02-22T20:15:32.873 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.47093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44356.log 2026-02-22T20:15:32.873 INFO:teuthology.orchestra.run.trial098.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47093.log.gz 2026-02-22T20:15:32.874 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.54634.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20801.log 2026-02-22T20:15:32.874 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54634.log.gz 2026-02-22T20:15:32.874 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.30689.log.gz 2026-02-22T20:15:32.874 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16517.log 2026-02-22T20:15:32.874 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44356.log: /var/log/ceph/cluster2-client.mirror.20801.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46046.log 2026-02-22T20:15:32.875 INFO:teuthology.orchestra.run.trial098.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.20801.log.gz 2026-02-22T20:15:32.875 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.16517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54855.log 2026-02-22T20:15:32.875 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16517.log.gz 2026-02-22T20:15:32.875 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44356.log.gz 2026-02-22T20:15:32.875 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46046.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21416.log 2026-02-22T20:15:32.875 INFO:teuthology.orchestra.run.trial098.stderr: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58546.log.gz 2026-02-22T20:15:32.875 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46046.log.gz 2026-02-22T20:15:32.875 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.54855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.20685.log 2026-02-22T20:15:32.876 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.21416.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34129.log 2026-02-22T20:15:32.876 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54855.log.gz 2026-02-22T20:15:32.876 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39751.log 2026-02-22T20:15:32.877 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.20685.log: 18.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.20685.log.gz/var/log/ceph/cluster1-client.mirror.34129.log: 2026-02-22T20:15:32.877 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47221.log 2026-02-22T20:15:32.877 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.34129.log.gz 2026-02-22T20:15:32.877 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.39751.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60615.log 2026-02-22T20:15:32.877 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.21416.log.gz 2026-02-22T20:15:32.877 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39751.log.gz 2026-02-22T20:15:32.877 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53385.log 2026-02-22T20:15:32.877 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47221.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47966.log 2026-02-22T20:15:32.878 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60615.log: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25704.log.gz 2026-02-22T20:15:32.878 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53385.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47221.log.gz 2026-02-22T20:15:32.878 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60615.log.gz 2026-02-22T20:15:32.878 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33490.log 2026-02-22T20:15:32.878 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.47966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28513.log 2026-02-22T20:15:32.878 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47966.log.gz 2026-02-22T20:15:32.878 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.53385.log.gz 2026-02-22T20:15:32.879 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33490.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20526.log 2026-02-22T20:15:32.879 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47410.log/var/log/ceph/cluster1-client.admin.28513.log: 2026-02-22T20:15:32.879 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.33490.log.gz 2026-02-22T20:15:32.879 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.20526.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.28513.log.gz 2026-02-22T20:15:32.879 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56002.log 2026-02-22T20:15:32.879 INFO:teuthology.orchestra.run.trial098.stderr: 57.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20526.log.gz 2026-02-22T20:15:32.880 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47410.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32847.log 2026-02-22T20:15:32.880 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47410.log.gz/var/log/ceph/cluster2-client.mirror.56002.log: 2026-02-22T20:15:32.880 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44734.log 2026-02-22T20:15:32.880 INFO:teuthology.orchestra.run.trial098.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56002.log.gz 2026-02-22T20:15:32.880 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32847.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37345.log 2026-02-22T20:15:32.881 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.32847.log.gz 2026-02-22T20:15:32.881 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34826.log 2026-02-22T20:15:32.881 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.37345.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.44734.log.gz 2026-02-22T20:15:32.881 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40682.log 2026-02-22T20:15:32.882 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.34826.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58189.log 2026-02-22T20:15:32.882 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.37345.log.gz 2026-02-22T20:15:32.882 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.40682.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30915.log 2026-02-22T20:15:32.882 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.58189.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.40682.log.gz 2026-02-22T20:15:32.882 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34826.log.gz 2026-02-22T20:15:32.882 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35861.log 2026-02-22T20:15:32.883 INFO:teuthology.orchestra.run.trial098.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58189.log.gz 2026-02-22T20:15:32.883 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24886.log 2026-02-22T20:15:32.883 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.30915.log.gz 2026-02-22T20:15:32.883 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53873.log 2026-02-22T20:15:32.883 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35861.log: /var/log/ceph/cluster2-client.mirror.24886.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24987.log 2026-02-22T20:15:32.883 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.35861.log.gz 2026-02-22T20:15:32.884 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.24886.log.gz 2026-02-22T20:15:32.884 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30451.log 2026-02-22T20:15:32.884 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53873.log.gz 2026-02-22T20:15:32.884 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24987.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35607.log 2026-02-22T20:15:32.884 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.24987.log.gz 2026-02-22T20:15:32.885 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30451.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23919.log 2026-02-22T20:15:32.885 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.30451.log.gz 2026-02-22T20:15:32.885 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.35607.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19155.log 2026-02-22T20:15:32.885 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35607.log.gz 2026-02-22T20:15:32.885 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23919.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46630.log 2026-02-22T20:15:32.886 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26249.log 2026-02-22T20:15:32.886 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19155.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19155.log.gz 2026-02-22T20:15:32.886 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46630.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.23919.log.gz 2026-02-22T20:15:32.886 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31751.log 2026-02-22T20:15:32.886 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46630.log.gz 2026-02-22T20:15:32.886 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.26249.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49607.log 2026-02-22T20:15:32.887 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.31751.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.26249.log.gz 2026-02-22T20:15:32.887 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43445.log 2026-02-22T20:15:32.887 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19338.log 2026-02-22T20:15:32.887 INFO:teuthology.orchestra.run.trial098.stderr: 87.7%/var/log/ceph/cluster2-client.admin.49607.log: -- replaced with /var/log/ceph/cluster2-client.admin.31751.log.gz 2026-02-22T20:15:32.887 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43445.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53907.log 2026-02-22T20:15:32.887 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.49607.log.gz 2026-02-22T20:15:32.887 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43445.log.gz 2026-02-22T20:15:32.888 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19338.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24072.log 2026-02-22T20:15:32.888 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19338.log.gz 2026-02-22T20:15:32.888 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.53907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52876.log 2026-02-22T20:15:32.888 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47284.log 2026-02-22T20:15:32.888 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.53907.log.gz 2026-02-22T20:15:32.889 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.24072.log: /var/log/ceph/cluster2-client.mirror.52876.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44539.log 2026-02-22T20:15:32.889 INFO:teuthology.orchestra.run.trial098.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.24072.log.gz 2026-02-22T20:15:32.889 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47284.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52876.log.gz 2026-02-22T20:15:32.889 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41356.log 2026-02-22T20:15:32.889 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49710.log 2026-02-22T20:15:32.890 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47284.log.gz 2026-02-22T20:15:32.890 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.41356.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44539.log.gz 2026-02-22T20:15:32.890 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53615.log 2026-02-22T20:15:32.890 INFO:teuthology.orchestra.run.trial098.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.41356.log.gz 2026-02-22T20:15:32.890 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49585.log 2026-02-22T20:15:32.890 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.49710.log: /var/log/ceph/cluster1-client.mirror.53615.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46935.log 2026-02-22T20:15:32.891 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.49710.log.gz 2026-02-22T20:15:32.891 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53615.log.gz 2026-02-22T20:15:32.891 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.49585.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19189.log 2026-02-22T20:15:32.891 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46935.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46935.log.gz 2026-02-22T20:15:32.891 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.49585.log.gz 2026-02-22T20:15:32.891 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27221.log 2026-02-22T20:15:32.892 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19189.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33409.log 2026-02-22T20:15:32.892 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19189.log.gz 2026-02-22T20:15:32.892 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.27221.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28638.log 2026-02-22T20:15:32.892 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24429.log 2026-02-22T20:15:32.893 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.33409.log.gz 2026-02-22T20:15:32.893 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.28638.log: gzip -5 --verbose -- 90.5% /var/log/ceph/cluster1-client.admin.55949.log 2026-02-22T20:15:32.893 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.27221.log.gz 2026-02-22T20:15:32.893 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.28638.log.gz 2026-02-22T20:15:32.893 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24429.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26690.log 2026-02-22T20:15:32.893 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54691.log 2026-02-22T20:15:32.894 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.55949.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.24429.log.gz 2026-02-22T20:15:32.894 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.55949.log.gz 2026-02-22T20:15:32.894 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.26690.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.54156.log 2026-02-22T20:15:32.894 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.26690.log.gz 2026-02-22T20:15:32.894 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.54691.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58723.log 2026-02-22T20:15:32.894 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.54691.log.gz 2026-02-22T20:15:32.895 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.54156.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52044.log 2026-02-22T20:15:32.895 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.54156.log.gz 2026-02-22T20:15:32.895 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46343.log 2026-02-22T20:15:32.895 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.58723.log: /var/log/ceph/cluster1-client.mirror.52044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.21363.log 2026-02-22T20:15:32.895 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.58723.log.gz 2026-02-22T20:15:32.895 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52044.log.gz 2026-02-22T20:15:32.896 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46343.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46893.log 2026-02-22T20:15:32.896 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46343.log.gz 2026-02-22T20:15:32.896 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.21363.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.54243.log 2026-02-22T20:15:32.896 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.21363.log.gz 2026-02-22T20:15:32.896 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46893.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25605.log 2026-02-22T20:15:32.897 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.54243.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.46893.log.gz 2026-02-22T20:15:32.897 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46385.log 2026-02-22T20:15:32.897 INFO:teuthology.orchestra.run.trial098.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.25125.log.gz 2026-02-22T20:15:32.897 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.25605.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43592.log 2026-02-22T20:15:32.897 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46385.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36224.log 2026-02-22T20:15:32.897 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.25605.log.gz 2026-02-22T20:15:32.897 INFO:teuthology.orchestra.run.trial098.stderr: 80.0% -- replaced with /var/log/ceph/cluster2-client.mirror.46385.log.gz 2026-02-22T20:15:32.898 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.43592.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31022.log 2026-02-22T20:15:32.898 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.43592.log.gz 2026-02-22T20:15:32.898 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46023.log 2026-02-22T20:15:32.898 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.36224.log: /var/log/ceph/cluster2-client.mirror.31022.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30763.log 2026-02-22T20:15:32.898 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.36224.log.gz 2026-02-22T20:15:32.898 INFO:teuthology.orchestra.run.trial098.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.54243.log.gz 2026-02-22T20:15:32.899 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.20743.log 2026-02-22T20:15:32.899 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31022.log.gz 2026-02-22T20:15:32.899 INFO:teuthology.orchestra.run.trial098.stderr:gzip 87.7% -5 --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.46023.log.gz /var/log/ceph/cluster2-client.mirror.27247.log 2026-02-22T20:15:32.899 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.899 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30763.log: /var/log/ceph/cluster1-client.mirror.2.20743.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53257.log 2026-02-22T20:15:32.899 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.30763.log.gz 2026-02-22T20:15:32.900 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43625.log 2026-02-22T20:15:32.900 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.27247.log: /var/log/ceph/cluster1-client.mirror.53257.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53257.log.gz 2026-02-22T20:15:32.900 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36801.log 2026-02-22T20:15:32.900 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27247.log.gz 2026-02-22T20:15:32.900 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53474.log 2026-02-22T20:15:32.900 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43625.log: /var/log/ceph/cluster1-client.admin.36801.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.25257.log 2026-02-22T20:15:32.901 INFO:teuthology.orchestra.run.trial098.stderr: 88.1%/var/log/ceph/cluster1-client.mirror.53474.log: -- replaced with /var/log/ceph/cluster1-client.admin.36801.log.gz 2026-02-22T20:15:32.901 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32481.log 2026-02-22T20:15:32.901 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.3.25257.log: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.20743.log.gz 2026-02-22T20:15:32.901 INFO:teuthology.orchestra.run.trial098.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.53474.log.gz 2026-02-22T20:15:32.901 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24186.log 2026-02-22T20:15:32.902 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.43625.log.gz 2026-02-22T20:15:32.902 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19187.log 2026-02-22T20:15:32.902 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32481.log: /var/log/ceph/cluster2-client.mirror.24186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35134.log 2026-02-22T20:15:32.902 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.32481.log.gz 2026-02-22T20:15:32.902 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24186.log.gz 2026-02-22T20:15:32.902 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster2-client.admin.19187.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.55296.log 2026-02-22T20:15:32.902 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19187.log.gz 2026-02-22T20:15:32.903 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35134.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16417.log 2026-02-22T20:15:32.903 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.35134.log.gz 2026-02-22T20:15:32.903 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.55296.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38181.log 2026-02-22T20:15:32.903 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.55296.log.gz 2026-02-22T20:15:32.903 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.16417.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26928.log 2026-02-22T20:15:32.903 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16417.log.gz 2026-02-22T20:15:32.904 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35914.log 2026-02-22T20:15:32.904 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.38181.log: /var/log/ceph/cluster2-client.admin.26928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49656.log 2026-02-22T20:15:32.904 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.26928.log.gz 2026-02-22T20:15:32.904 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32984.log 2026-02-22T20:15:32.905 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.35914.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.38181.log.gz 2026-02-22T20:15:32.905 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.49656.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57308.log 2026-02-22T20:15:32.905 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.admin.49656.log.gz 2026-02-22T20:15:32.905 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.35914.log.gz 2026-02-22T20:15:32.905 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.32984.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24166.log 2026-02-22T20:15:32.905 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.57308.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.32984.log.gz 2026-02-22T20:15:32.905 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44477.log 2026-02-22T20:15:32.906 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57308.log.gz 2026-02-22T20:15:32.906 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24166.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44209.log 2026-02-22T20:15:32.906 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44477.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.24166.log.gz 2026-02-22T20:15:32.906 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19129.log 2026-02-22T20:15:32.906 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.44477.log.gz 2026-02-22T20:15:32.906 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44209.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21258.log 2026-02-22T20:15:32.907 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58033.log 2026-02-22T20:15:32.907 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19129.log: /var/log/ceph/cluster1-client.mirror.21258.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.44209.log.gz 2026-02-22T20:15:32.907 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19129.log.gz 2026-02-22T20:15:32.907 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.21258.log.gz 2026-02-22T20:15:32.907 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32382.log 2026-02-22T20:15:32.908 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.58033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59963.log 2026-02-22T20:15:32.908 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.32382.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.58033.log.gz 2026-02-22T20:15:32.908 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24678.log 2026-02-22T20:15:32.908 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.59963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29437.log 2026-02-22T20:15:32.908 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.32382.log.gz 2026-02-22T20:15:32.909 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.24678.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59963.log.gz 2026-02-22T20:15:32.909 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46608.log 2026-02-22T20:15:32.909 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.24678.log.gz 2026-02-22T20:15:32.909 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.29437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36882.log 2026-02-22T20:15:32.909 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46608.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.29437.log.gz 2026-02-22T20:15:32.909 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51096.log 2026-02-22T20:15:32.909 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46608.log.gz 2026-02-22T20:15:32.910 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36882.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55112.log 2026-02-22T20:15:32.910 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55850.log 2026-02-22T20:15:32.910 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.36882.log.gz 2026-02-22T20:15:32.910 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.51096.log: /var/log/ceph/cluster1-client.admin.55112.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47515.log 2026-02-22T20:15:32.911 INFO:teuthology.orchestra.run.trial098.stderr: 89.0% 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51096.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.55112.log.gz 2026-02-22T20:15:32.911 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.911 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58587.log 2026-02-22T20:15:32.911 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.55850.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.55850.log.gz 2026-02-22T20:15:32.911 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47515.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47070.log 2026-02-22T20:15:32.911 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47515.log.gz 2026-02-22T20:15:32.911 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.58587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39290.log 2026-02-22T20:15:32.912 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.47070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51070.log 2026-02-22T20:15:32.912 INFO:teuthology.orchestra.run.trial098.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58587.log.gz 2026-02-22T20:15:32.912 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47070.log.gz 2026-02-22T20:15:32.912 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35397.log 2026-02-22T20:15:32.912 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.39290.log: /var/log/ceph/cluster1-client.admin.51070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52076.log 2026-02-22T20:15:32.912 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.39290.log.gz 2026-02-22T20:15:32.913 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.51070.log.gz 2026-02-22T20:15:32.913 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.35397.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52457.log 2026-02-22T20:15:32.913 INFO:teuthology.orchestra.run.trial098.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58943.log.gz 2026-02-22T20:15:32.913 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16393.log 2026-02-22T20:15:32.913 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52076.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.35397.log.gz 2026-02-22T20:15:32.913 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52457.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52076.log.gz 2026-02-22T20:15:32.913 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45821.log 2026-02-22T20:15:32.914 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.16393.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16393.log.gz 2026-02-22T20:15:32.914 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52457.log.gz 2026-02-22T20:15:32.914 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53068.log 2026-02-22T20:15:32.914 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44379.log 2026-02-22T20:15:32.914 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45821.log.gz 2026-02-22T20:15:32.914 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43291.log 2026-02-22T20:15:32.915 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45755.log 2026-02-22T20:15:32.915 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53068.log.gz 2026-02-22T20:15:32.915 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44379.log: /var/log/ceph/cluster2-client.mirror.43291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31809.log 2026-02-22T20:15:32.915 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43291.log.gz 2026-02-22T20:15:32.915 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45755.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44379.log.gz 2026-02-22T20:15:32.915 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55323.log 2026-02-22T20:15:32.915 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.45755.log.gz 2026-02-22T20:15:32.916 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.31809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53408.log 2026-02-22T20:15:32.916 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.55323.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29045.log 2026-02-22T20:15:32.916 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51286.log 2026-02-22T20:15:32.916 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.55323.log.gz/var/log/ceph/cluster2-client.mirror.53408.log: 2026-02-22T20:15:32.916 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31809.log.gz 2026-02-22T20:15:32.917 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.29045.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28330.log 2026-02-22T20:15:32.917 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.29045.log.gz 2026-02-22T20:15:32.917 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.53408.log.gz 2026-02-22T20:15:32.917 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.51286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43683.log 2026-02-22T20:15:32.918 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28330.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31209.log 2026-02-22T20:15:32.918 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.28330.log.gz 2026-02-22T20:15:32.918 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39906.log 2026-02-22T20:15:32.918 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31209.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36908.log 2026-02-22T20:15:32.918 INFO:teuthology.orchestra.run.trial098.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51286.log.gz 2026-02-22T20:15:32.919 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43683.log.gz 2026-02-22T20:15:32.919 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.31209.log.gz 2026-02-22T20:15:32.919 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.39906.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45183.log 2026-02-22T20:15:32.919 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.36908.log: gzip -5 --verbose 89.3% -- /var/log/ceph/cluster2-client.admin.50318.log -- replaced with /var/log/ceph/cluster1-client.mirror.39906.log.gz 2026-02-22T20:15:32.919 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.919 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.36908.log.gz 2026-02-22T20:15:32.920 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44400.log 2026-02-22T20:15:32.920 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.50318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26564.log 2026-02-22T20:15:32.920 INFO:teuthology.orchestra.run.trial098.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45183.log.gz 2026-02-22T20:15:32.920 INFO:teuthology.orchestra.run.trial098.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.25257.log.gz 2026-02-22T20:15:32.920 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44400.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32226.log 2026-02-22T20:15:32.920 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.50318.log.gz 2026-02-22T20:15:32.921 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.26564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29980.log 2026-02-22T20:15:32.921 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44400.log.gz 2026-02-22T20:15:32.921 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49099.log 2026-02-22T20:15:32.921 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.32226.log: /var/log/ceph/cluster2-client.mirror.29980.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.26564.log.gz 2026-02-22T20:15:32.921 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33184.log 2026-02-22T20:15:32.921 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29980.log.gz 2026-02-22T20:15:32.921 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.32226.log.gz 2026-02-22T20:15:32.921 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.49099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41116.log 2026-02-22T20:15:32.922 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.33184.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45346.log 2026-02-22T20:15:32.922 INFO:teuthology.orchestra.run.trial098.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.admin.49099.log.gz 2026-02-22T20:15:32.922 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.33184.log.gz 2026-02-22T20:15:32.922 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.41116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53987.log 2026-02-22T20:15:32.922 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.41116.log.gz 2026-02-22T20:15:32.922 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45346.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28151.log 2026-02-22T20:15:32.922 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45346.log.gz 2026-02-22T20:15:32.923 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53987.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40991.log 2026-02-22T20:15:32.923 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.50538.log 2026-02-22T20:15:32.923 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.28151.log: /var/log/ceph/cluster1-client.admin.40991.log: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53987.log.gz 2026-02-22T20:15:32.923 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47117.log 2026-02-22T20:15:32.923 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.28151.log.gz 2026-02-22T20:15:32.923 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.40991.log.gz 2026-02-22T20:15:32.924 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.50538.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24849.log 2026-02-22T20:15:32.924 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47117.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31911.log 2026-02-22T20:15:32.924 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24849.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34729.log 2026-02-22T20:15:32.925 INFO:teuthology.orchestra.run.trial098.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.mirror.24849.log.gz 2026-02-22T20:15:32.925 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47117.log.gz 2026-02-22T20:15:32.925 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40482.log 2026-02-22T20:15:32.925 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.31911.log: /var/log/ceph/cluster2-client.admin.34729.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.55404.log 2026-02-22T20:15:32.925 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.31911.log.gz 2026-02-22T20:15:32.925 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.34729.log.gz 2026-02-22T20:15:32.925 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.40482.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46402.log 2026-02-22T20:15:32.926 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.0.55404.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40482.log.gz 2026-02-22T20:15:32.926 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46068.log 2026-02-22T20:15:32.926 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46402.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16300.log 2026-02-22T20:15:32.927 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46068.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28486.log 2026-02-22T20:15:32.927 INFO:teuthology.orchestra.run.trial098.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46402.log.gz 2026-02-22T20:15:32.927 INFO:teuthology.orchestra.run.trial098.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.50538.log.gz 87.9% 2026-02-22T20:15:32.927 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.46068.log.gz 2026-02-22T20:15:32.927 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31182.log 2026-02-22T20:15:32.927 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.16300.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16300.log.gz 2026-02-22T20:15:32.927 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.28486.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47179.log 2026-02-22T20:15:32.927 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.28486.log.gz 2026-02-22T20:15:32.928 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31182.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27794.log 2026-02-22T20:15:32.928 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.31182.log.gz 2026-02-22T20:15:32.928 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47179.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28178.log 2026-02-22T20:15:32.928 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47179.log.gz 2026-02-22T20:15:32.928 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27794.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53518.log 2026-02-22T20:15:32.929 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39681.log 2026-02-22T20:15:32.929 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.28178.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.27794.log.gz 2026-02-22T20:15:32.929 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53518.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53518.log.gz 2026-02-22T20:15:32.929 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41682.log 2026-02-22T20:15:32.929 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.28178.log.gz 2026-02-22T20:15:32.929 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.39681.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35344.log 2026-02-22T20:15:32.930 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.39681.log.gz 2026-02-22T20:15:32.930 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.41682.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49788.log 2026-02-22T20:15:32.930 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.41682.log.gz 2026-02-22T20:15:32.930 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23424.log 2026-02-22T20:15:32.930 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35344.log: /var/log/ceph/cluster1-client.admin.49788.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27946.log 2026-02-22T20:15:32.930 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.35344.log.gz 2026-02-22T20:15:32.931 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23424.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.49788.log.gz 2026-02-22T20:15:32.931 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52835.log 2026-02-22T20:15:32.931 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.23424.log.gz 2026-02-22T20:15:32.931 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27946.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20599.log 2026-02-22T20:15:32.931 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.27946.log.gz 2026-02-22T20:15:32.931 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27581.log 2026-02-22T20:15:32.932 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.52835.log.gz 2026-02-22T20:15:32.932 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.20599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49475.log 2026-02-22T20:15:32.932 INFO:teuthology.orchestra.run.trial098.stderr: 57.5% -- replaced with /var/log/ceph/cluster2-client.admin.20599.log.gz 2026-02-22T20:15:32.932 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27581.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23451.log 2026-02-22T20:15:32.932 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.49475.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.27581.log.gz 2026-02-22T20:15:32.932 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37975.log 2026-02-22T20:15:32.933 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23451.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48068.log 2026-02-22T20:15:32.933 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.23451.log.gz 2026-02-22T20:15:32.933 INFO:teuthology.orchestra.run.trial098.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.49475.log.gz 2026-02-22T20:15:32.933 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.37975.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49924.log 2026-02-22T20:15:32.933 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.37975.log.gz 2026-02-22T20:15:32.933 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.21698.log 2026-02-22T20:15:32.934 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.48068.log: /var/log/ceph/cluster2-client.mirror.49924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52161.log 2026-02-22T20:15:32.934 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.48068.log.gz 2026-02-22T20:15:32.934 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.21698.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57733.log 2026-02-22T20:15:32.934 INFO:teuthology.orchestra.run.trial098.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49924.log.gz 2026-02-22T20:15:32.934 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50933.log 2026-02-22T20:15:32.934 INFO:teuthology.orchestra.run.trial098.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52161.log.gz 2026-02-22T20:15:32.935 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28719.log 2026-02-22T20:15:32.935 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.57733.log: /var/log/ceph/cluster2-client.admin.50933.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57733.log.gz 2026-02-22T20:15:32.935 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35523.log 2026-02-22T20:15:32.935 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.admin.50933.log.gz 2026-02-22T20:15:32.935 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33436.log 2026-02-22T20:15:32.935 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.28719.log.gz 2026-02-22T20:15:32.936 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35523.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52707.log 2026-02-22T20:15:32.936 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33436.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33750.log 2026-02-22T20:15:32.936 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.35523.log.gz 2026-02-22T20:15:32.936 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.52707.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.52707.log.gz 2026-02-22T20:15:32.937 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.33436.log.gz 2026-02-22T20:15:32.937 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47804.log 2026-02-22T20:15:32.937 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.33750.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30346.log 2026-02-22T20:15:32.937 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.33750.log.gz 2026-02-22T20:15:32.937 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47804.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45673.log 2026-02-22T20:15:32.938 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.30346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39263.log 2026-02-22T20:15:32.938 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.47804.log.gz 2026-02-22T20:15:32.938 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45673.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30346.log.gz 2026-02-22T20:15:32.938 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35834.log 2026-02-22T20:15:32.938 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.39263.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49683.log 2026-02-22T20:15:32.939 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.39263.log.gz 2026-02-22T20:15:32.939 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.35834.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43565.log 2026-02-22T20:15:32.939 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.35834.log.gz 2026-02-22T20:15:32.939 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.49683.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39111.log 2026-02-22T20:15:32.939 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.43565.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.49683.log.gz 2026-02-22T20:15:32.940 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41537.log 2026-02-22T20:15:32.940 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.43565.log.gz 2026-02-22T20:15:32.940 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.39111.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.50606.log 2026-02-22T20:15:32.940 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.39111.log.gz 2026-02-22T20:15:32.940 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.41537.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53162.log 2026-02-22T20:15:32.941 INFO:teuthology.orchestra.run.trial098.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45673.log.gz 2026-02-22T20:15:32.941 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.50606.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.50606.log.gz 2026-02-22T20:15:32.941 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50068.log 2026-02-22T20:15:32.941 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41537.log.gz 2026-02-22T20:15:32.941 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.53162.log: --verbose -- /var/log/ceph/cluster2-client.mirror.46688.log 2026-02-22T20:15:32.941 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.50068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33113.log 2026-02-22T20:15:32.942 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46208.log 2026-02-22T20:15:32.942 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27713.log 2026-02-22T20:15:32.942 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33113.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46688.log.gz 2026-02-22T20:15:32.942 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46208.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41997.log 2026-02-22T20:15:32.943 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.33113.log.gz 2026-02-22T20:15:32.943 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.27713.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46208.log.gz 2026-02-22T20:15:32.943 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41302.log 2026-02-22T20:15:32.943 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.27713.log.gz 2026-02-22T20:15:32.943 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.41997.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24472.log 2026-02-22T20:15:32.944 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41997.log.gz 2026-02-22T20:15:32.944 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41835.log 2026-02-22T20:15:32.944 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.41302.log.gz 2026-02-22T20:15:32.944 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.24472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46126.log 2026-02-22T20:15:32.944 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.24472.log.gz 2026-02-22T20:15:32.944 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.28070.log 2026-02-22T20:15:32.945 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46126.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.41835.log.gz 2026-02-22T20:15:32.945 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37814.log 2026-02-22T20:15:32.945 INFO:teuthology.orchestra.run.trial098.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53162.log.gz 2026-02-22T20:15:32.945 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.28070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43059.log 2026-02-22T20:15:32.945 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.46126.log.gz 2026-02-22T20:15:32.945 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.28070.log.gz 2026-02-22T20:15:32.945 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.25196.log 2026-02-22T20:15:32.946 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.37814.log: /var/log/ceph/cluster1-client.mirror.43059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44273.log 2026-02-22T20:15:32.946 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.2.25196.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50068.log.gz 2026-02-22T20:15:32.946 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34955.log 2026-02-22T20:15:32.946 INFO:teuthology.orchestra.run.trial098.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37814.log.gz 2026-02-22T20:15:32.946 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44273.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40637.log 2026-02-22T20:15:32.947 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34955.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44273.log.gz 2026-02-22T20:15:32.947 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.57372.log 2026-02-22T20:15:32.947 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.40637.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.34955.log.gz 2026-02-22T20:15:32.947 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44898.log 2026-02-22T20:15:32.948 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.2.57372.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.40637.log.gz 2026-02-22T20:15:32.948 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.57372.log.gz 2026-02-22T20:15:32.948 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36410.log 2026-02-22T20:15:32.948 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44898.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26195.log 2026-02-22T20:15:32.948 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36410.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.44898.log.gz 2026-02-22T20:15:32.948 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.55395.log 2026-02-22T20:15:32.948 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.36410.log.gz 2026-02-22T20:15:32.948 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.26195.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28665.log 2026-02-22T20:15:32.948 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.26195.log.gz 2026-02-22T20:15:32.949 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.0.55395.log: 20.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.55395.log.gz 2026-02-22T20:15:32.949 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40937.log 2026-02-22T20:15:32.949 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.28665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46228.log 2026-02-22T20:15:32.949 INFO:teuthology.orchestra.run.trial098.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.43059.log.gz 2026-02-22T20:15:32.950 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.40937.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.28665.log.gz 2026-02-22T20:15:32.950 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60329.log 2026-02-22T20:15:32.950 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.40937.log.gz 2026-02-22T20:15:32.950 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46228.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27639.log 2026-02-22T20:15:32.950 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60329.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46228.log.gz 2026-02-22T20:15:32.950 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52308.log 2026-02-22T20:15:32.951 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60329.log.gz 2026-02-22T20:15:32.951 INFO:teuthology.orchestra.run.trial098.stderr: 95.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.57339.log.gz 2026-02-22T20:15:32.951 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.27639.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32793.log 2026-02-22T20:15:32.951 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52308.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55231.log 2026-02-22T20:15:32.951 INFO:teuthology.orchestra.run.trial098.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52308.log.gz 2026-02-22T20:15:32.951 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32793.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20162.log 2026-02-22T20:15:32.952 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.27639.log.gz 2026-02-22T20:15:32.952 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.55231.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.32793.log.gz 2026-02-22T20:15:32.952 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40557.log 2026-02-22T20:15:32.952 INFO:teuthology.orchestra.run.trial098.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.55231.log.gz 2026-02-22T20:15:32.952 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.20162.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32329.log 2026-02-22T20:15:32.952 INFO:teuthology.orchestra.run.trial098.stderr: 58.0% -- replaced with /var/log/ceph/cluster1-client.mirror.20162.log.gz 2026-02-22T20:15:32.953 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.40557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30844.log 2026-02-22T20:15:32.953 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32329.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.40557.log.gz 2026-02-22T20:15:32.953 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19342.log 2026-02-22T20:15:32.953 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.32329.log.gz 2026-02-22T20:15:32.953 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44456.log 2026-02-22T20:15:32.954 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19342.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47743.log 2026-02-22T20:15:32.954 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19342.log.gz 2026-02-22T20:15:32.954 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.30844.log.gz 2026-02-22T20:15:32.954 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44456.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52855.log 2026-02-22T20:15:32.954 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44456.log.gz 2026-02-22T20:15:32.954 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47743.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30790.log 2026-02-22T20:15:32.955 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38421.log 2026-02-22T20:15:32.955 INFO:teuthology.orchestra.run.trial098.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47743.log.gz 2026-02-22T20:15:32.955 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.30790.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.54258.log 2026-02-22T20:15:32.955 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.30790.log.gz 2026-02-22T20:15:32.956 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.38421.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60459.log 2026-02-22T20:15:32.956 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52855.log.gz 2026-02-22T20:15:32.956 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.54258.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22992.log 2026-02-22T20:15:32.956 INFO:teuthology.orchestra.run.trial098.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.54258.log.gz 2026-02-22T20:15:32.956 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.38421.log.gz 2026-02-22T20:15:32.956 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60459.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36855.log 2026-02-22T20:15:32.956 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60459.log.gz 2026-02-22T20:15:32.957 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22992.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22508.log 2026-02-22T20:15:32.957 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.22992.log.gz 2026-02-22T20:15:32.957 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26847.log 2026-02-22T20:15:32.957 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.36855.log.gz 2026-02-22T20:15:32.957 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22508.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47787.log 2026-02-22T20:15:32.958 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5/var/log/ceph/cluster1-client.admin.26847.log: --verbose -- /var/log/ceph/cluster1-client.admin.26222.log 2026-02-22T20:15:32.958 INFO:teuthology.orchestra.run.trial098.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.22508.log.gz 2026-02-22T20:15:32.958 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47787.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47787.log.gz 2026-02-22T20:15:32.958 INFO:teuthology.orchestra.run.trial098.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.26847.log.gz 2026-02-22T20:15:32.958 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34473.log 2026-02-22T20:15:32.958 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.26222.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22149.log 2026-02-22T20:15:32.959 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.34473.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.26222.log.gz 2026-02-22T20:15:32.959 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48808.log 2026-02-22T20:15:32.959 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34473.log.gz 2026-02-22T20:15:32.959 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.22149.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22125.log 2026-02-22T20:15:32.960 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.22149.log.gz 2026-02-22T20:15:32.960 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.48808.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41275.log 2026-02-22T20:15:32.960 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.22125.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.48808.log.gz 2026-02-22T20:15:32.960 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37788.log 2026-02-22T20:15:32.960 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.22125.log.gz 2026-02-22T20:15:32.961 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.41275.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30322.log 2026-02-22T20:15:32.961 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.41275.log.gz 2026-02-22T20:15:32.961 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.37788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39725.log 2026-02-22T20:15:32.961 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.30322.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.37788.log.gz 2026-02-22T20:15:32.961 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39880.log 2026-02-22T20:15:32.961 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30322.log.gz 2026-02-22T20:15:32.962 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.39725.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54838.log 2026-02-22T20:15:32.962 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.39880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27295.log 2026-02-22T20:15:32.962 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.54838.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.39880.log.gz 2026-02-22T20:15:32.962 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.39725.log.gz 2026-02-22T20:15:32.962 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32747.log 2026-02-22T20:15:32.963 INFO:teuthology.orchestra.run.trial098.stderr: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54838.log.gz 2026-02-22T20:15:32.963 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.27295.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31307.log 2026-02-22T20:15:32.963 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.32747.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.27295.log.gz 2026-02-22T20:15:32.963 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34007.log 2026-02-22T20:15:32.964 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31307.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32747.log.gz 2026-02-22T20:15:32.964 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33828.log 2026-02-22T20:15:32.964 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.31307.log.gz 2026-02-22T20:15:32.964 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.34007.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29410.log 2026-02-22T20:15:32.964 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33828.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.34007.log.gz 2026-02-22T20:15:32.964 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.25158.log 2026-02-22T20:15:32.965 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38520.log 2026-02-22T20:15:32.965 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.33828.log.gz 2026-02-22T20:15:32.965 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.29410.log: /var/log/ceph/cluster2-client.mirror.2.25158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38290.log 2026-02-22T20:15:32.965 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.2.25158.log.gz 2026-02-22T20:15:32.965 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.29410.log.gz 2026-02-22T20:15:32.965 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.38520.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55156.log 2026-02-22T20:15:32.965 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.38520.log.gz 2026-02-22T20:15:32.966 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.38290.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31361.log 2026-02-22T20:15:32.966 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46708.log 2026-02-22T20:15:32.966 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38290.log.gz 2026-02-22T20:15:32.966 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.55156.log: /var/log/ceph/cluster2-client.admin.31361.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29787.log 2026-02-22T20:15:32.966 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.31361.log.gz 2026-02-22T20:15:32.966 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.55156.log.gz 2026-02-22T20:15:32.967 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46708.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23298.log 2026-02-22T20:15:32.967 INFO:teuthology.orchestra.run.trial098.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.2.25196.log.gz 2026-02-22T20:15:32.967 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.29787.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52288.log 2026-02-22T20:15:32.967 INFO:teuthology.orchestra.run.trial098.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46708.log.gz 2026-02-22T20:15:32.967 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23298.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.29787.log.gz 2026-02-22T20:15:32.967 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31670.log 2026-02-22T20:15:32.967 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.23298.log.gz 2026-02-22T20:15:32.968 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28094.log 2026-02-22T20:15:32.968 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52288.log.gz 2026-02-22T20:15:32.968 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.31670.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24389.log 2026-02-22T20:15:32.968 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.31670.log.gz 2026-02-22T20:15:32.969 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28094.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52740.log 2026-02-22T20:15:32.969 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24389.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.28094.log.gz 2026-02-22T20:15:32.969 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43517.log 2026-02-22T20:15:32.969 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24389.log.gz 2026-02-22T20:15:32.969 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52740.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57976.log 2026-02-22T20:15:32.970 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30296.log 2026-02-22T20:15:32.970 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36126.log 2026-02-22T20:15:32.970 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.43517.log.gz 2026-02-22T20:15:32.970 INFO:teuthology.orchestra.run.trial098.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52740.log.gz 2026-02-22T20:15:32.970 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.57976.log: /var/log/ceph/cluster2-client.mirror.30296.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60563.log 2026-02-22T20:15:32.970 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57976.log.gz 2026-02-22T20:15:32.971 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36126.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30296.log.gz 2026-02-22T20:15:32.971 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20475.log 2026-02-22T20:15:32.971 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.36126.log.gz 2026-02-22T20:15:32.971 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49864.log 2026-02-22T20:15:32.971 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60563.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60563.log.gz 2026-02-22T20:15:32.972 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.20475.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45499.log 2026-02-22T20:15:32.972 INFO:teuthology.orchestra.run.trial098.stderr: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.20475.log.gz 2026-02-22T20:15:32.972 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.49864.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45961.log 2026-02-22T20:15:32.972 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45499.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.49864.log.gz 2026-02-22T20:15:32.972 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44253.log 2026-02-22T20:15:32.972 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53649.log 2026-02-22T20:15:32.973 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45961.log: /var/log/ceph/cluster2-client.mirror.44253.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42224.log 2026-02-22T20:15:32.973 INFO:teuthology.orchestra.run.trial098.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.45499.log.gz 2026-02-22T20:15:32.973 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44253.log.gz 2026-02-22T20:15:32.973 INFO:teuthology.orchestra.run.trial098.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45961.log.gz 2026-02-22T20:15:32.973 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59921.log 2026-02-22T20:15:32.973 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.53649.log: /var/log/ceph/cluster1-client.admin.42224.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.53649.log.gz 2026-02-22T20:15:32.974 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49170.log 2026-02-22T20:15:32.974 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.42224.log.gz 2026-02-22T20:15:32.974 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.59921.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43271.log 2026-02-22T20:15:32.974 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.49170.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41227.log 2026-02-22T20:15:32.974 INFO:teuthology.orchestra.run.trial098.stderr: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49170.log.gz 2026-02-22T20:15:32.974 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.59921.log.gz 2026-02-22T20:15:32.974 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.43271.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36021.log 2026-02-22T20:15:32.975 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.41227.log: 87.8% 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41227.log.gz 2026-02-22T20:15:32.975 INFO:teuthology.orchestra.run.trial098.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.43271.log.gz 2026-02-22T20:15:32.975 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20001.log 2026-02-22T20:15:32.975 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.36021.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27820.log 2026-02-22T20:15:32.975 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36021.log.gz 2026-02-22T20:15:32.976 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.20001.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54557.log 2026-02-22T20:15:32.976 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20001.log.gz 2026-02-22T20:15:32.976 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.27820.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26799.log 2026-02-22T20:15:32.976 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.54557.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45548.log 2026-02-22T20:15:32.976 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.27820.log.gz 2026-02-22T20:15:32.977 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.26799.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54557.log.gz 2026-02-22T20:15:32.977 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35970.log 2026-02-22T20:15:32.977 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.26799.log.gz 2026-02-22T20:15:32.977 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45548.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26408.log 2026-02-22T20:15:32.977 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.35970.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19619.log 2026-02-22T20:15:32.978 INFO:teuthology.orchestra.run.trial098.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45548.log.gz 2026-02-22T20:15:32.978 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.26408.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34207.log 2026-02-22T20:15:32.978 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31836.log 2026-02-22T20:15:32.978 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.26408.log.gz 2026-02-22T20:15:32.978 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19619.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.35970.log.gz 2026-02-22T20:15:32.978 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19619.log.gz 2026-02-22T20:15:32.978 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34207.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22632.log 2026-02-22T20:15:32.978 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.34207.log.gz 2026-02-22T20:15:32.979 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.31836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.55536.log 2026-02-22T20:15:32.979 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31836.log.gz 2026-02-22T20:15:32.979 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22632.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27527.log 2026-02-22T20:15:32.979 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.22632.log.gz 2026-02-22T20:15:32.979 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.3.55536.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53709.log 2026-02-22T20:15:32.980 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.27527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54896.log 2026-02-22T20:15:32.980 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.27527.log.gz 2026-02-22T20:15:32.980 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.53709.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45162.log 2026-02-22T20:15:32.981 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.54896.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47137.log 2026-02-22T20:15:32.981 INFO:teuthology.orchestra.run.trial098.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.admin.54896.log.gz 2026-02-22T20:15:32.981 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45162.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53233.log 2026-02-22T20:15:32.981 INFO:teuthology.orchestra.run.trial098.stderr: 95.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.55404.log.gz 2026-02-22T20:15:32.981 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47137.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58822.log 2026-02-22T20:15:32.982 INFO:teuthology.orchestra.run.trial098.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53709.log.gz 2026-02-22T20:15:32.982 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34033.log 2026-02-22T20:15:32.982 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.53233.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47137.log.gz 2026-02-22T20:15:32.982 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.58822.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53233.log.gz 2026-02-22T20:15:32.982 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.58822.log.gz 2026-02-22T20:15:32.982 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60355.log 2026-02-22T20:15:32.982 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.34033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28303.log 2026-02-22T20:15:32.983 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60355.log: gzip 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34033.log.gz -5 2026-02-22T20:15:32.983 INFO:teuthology.orchestra.run.trial098.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.22686.log 2026-02-22T20:15:32.983 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.60355.log.gz 2026-02-22T20:15:32.983 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.28303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45250.log 2026-02-22T20:15:32.983 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.admin.28303.log.gz 2026-02-22T20:15:32.984 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22686.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40709.log 2026-02-22T20:15:32.984 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.22686.log.gz 2026-02-22T20:15:32.984 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45250.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45632.log 2026-02-22T20:15:32.984 INFO:teuthology.orchestra.run.trial098.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.21698.log.gz 2026-02-22T20:15:32.984 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.40709.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46320.log 2026-02-22T20:15:32.984 INFO:teuthology.orchestra.run.trial098.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45162.log.gz 2026-02-22T20:15:32.984 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45250.log.gz 2026-02-22T20:15:32.985 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.45632.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.40709.log.gz 2026-02-22T20:15:32.985 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35371.log 2026-02-22T20:15:32.985 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45632.log.gz 2026-02-22T20:15:32.985 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46320.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60076.log 2026-02-22T20:15:32.985 INFO:teuthology.orchestra.run.trial098.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.3.55536.log.gz 2026-02-22T20:15:32.985 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46320.log.gz 2026-02-22T20:15:32.985 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35371.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.48398.log 2026-02-22T20:15:32.986 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.35371.log.gz 2026-02-22T20:15:32.986 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.60076.log: gzip -5 --verbose -- 0.0% /var/log/ceph/cluster2-client.mirror.47494.log -- replaced with /var/log/ceph/cluster2-client.admin.60076.log.gz 2026-02-22T20:15:32.986 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:32.986 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.3.48398.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40910.log 2026-02-22T20:15:32.987 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.47494.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35581.log 2026-02-22T20:15:32.987 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47494.log.gz 2026-02-22T20:15:32.987 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.40910.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33265.log 2026-02-22T20:15:32.987 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35107.log 2026-02-22T20:15:32.987 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.35581.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.40910.log.gz 2026-02-22T20:15:32.988 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.33265.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31860.log 2026-02-22T20:15:32.988 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.33265.log.gz 2026-02-22T20:15:32.988 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35581.log.gz 2026-02-22T20:15:32.988 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.35107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44147.log 2026-02-22T20:15:32.988 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.31860.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.35107.log.gz 2026-02-22T20:15:32.988 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23271.log 2026-02-22T20:15:32.989 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.31860.log.gz 2026-02-22T20:15:32.989 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44147.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25552.log 2026-02-22T20:15:32.989 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23271.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58771.log 2026-02-22T20:15:32.989 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44147.log.gz 2026-02-22T20:15:32.990 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.25552.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45370.log 2026-02-22T20:15:32.990 INFO:teuthology.orchestra.run.trial098.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.25552.log.gz 2026-02-22T20:15:32.990 INFO:teuthology.orchestra.run.trial098.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.admin.23271.log.gz 2026-02-22T20:15:32.990 INFO:teuthology.orchestra.run.trial098.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.3.48398.log.gz 2026-02-22T20:15:32.990 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.58771.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27103.log 2026-02-22T20:15:32.990 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58771.log.gz 2026-02-22T20:15:32.990 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.45370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57904.log 2026-02-22T20:15:32.991 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.27103.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.45370.log.gz 2026-02-22T20:15:32.991 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.27103.log.gz 2026-02-22T20:15:32.991 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34602.log 2026-02-22T20:15:32.991 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24866.log 2026-02-22T20:15:32.991 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.57904.log: /var/log/ceph/cluster2-client.admin.34602.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.34602.log.gz 2026-02-22T20:15:32.992 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24627.log 2026-02-22T20:15:32.992 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57904.log.gz 2026-02-22T20:15:32.992 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19894.log 2026-02-22T20:15:32.992 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24866.log: /var/log/ceph/cluster2-client.mirror.24627.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47003.log 2026-02-22T20:15:32.992 INFO:teuthology.orchestra.run.trial098.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.24866.log.gz 2026-02-22T20:15:32.993 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.19894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27409.log 2026-02-22T20:15:32.993 INFO:teuthology.orchestra.run.trial098.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24627.log.gz 2026-02-22T20:15:32.993 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19894.log.gz 2026-02-22T20:15:32.993 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.47003.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.55432.log 2026-02-22T20:15:32.993 INFO:teuthology.orchestra.run.trial098.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47003.log.gz 2026-02-22T20:15:32.993 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.27409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41484.log 2026-02-22T20:15:32.994 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.1.55432.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58627.log 2026-02-22T20:15:32.994 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.41484.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22541.log 2026-02-22T20:15:32.994 INFO:teuthology.orchestra.run.trial098.stderr: 87.7%/var/log/ceph/cluster2-client.mirror.58627.log: -- replaced with /var/log/ceph/cluster2-client.admin.41484.log.gz 2026-02-22T20:15:32.994 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21282.log 2026-02-22T20:15:32.995 INFO:teuthology.orchestra.run.trial098.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.27409.log.gz 2026-02-22T20:15:32.995 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.22541.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38266.log 2026-02-22T20:15:32.995 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.21282.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28016.log 2026-02-22T20:15:32.995 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.21282.log.gz 2026-02-22T20:15:32.996 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.38266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.21675.log 2026-02-22T20:15:32.996 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38266.log.gz 2026-02-22T20:15:32.996 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39853.log 2026-02-22T20:15:32.996 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.28016.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.22541.log.gz 2026-02-22T20:15:32.996 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.21675.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.21675.log.gz 2026-02-22T20:15:32.996 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58340.log 2026-02-22T20:15:32.997 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster2-client.admin.39853.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.23865.log 2026-02-22T20:15:32.997 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.58340.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28016.log.gz 2026-02-22T20:15:32.997 INFO:teuthology.orchestra.run.trial098.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.58340.log.gz 2026-02-22T20:15:32.997 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.39853.log.gz 2026-02-22T20:15:32.997 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39951.log 2026-02-22T20:15:32.997 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.23865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.48275.log 2026-02-22T20:15:32.997 INFO:teuthology.orchestra.run.trial098.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58627.log.gz 2026-02-22T20:15:32.998 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.39951.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52528.log 2026-02-22T20:15:32.998 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.23865.log.gz 2026-02-22T20:15:32.998 INFO:teuthology.orchestra.run.trial098.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.1.55432.log.gz 2026-02-22T20:15:32.998 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.0.48275.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.39951.log.gz 2026-02-22T20:15:32.998 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46585.log 2026-02-22T20:15:32.998 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.52528.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40218.log 2026-02-22T20:15:32.999 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.46585.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52920.log 2026-02-22T20:15:32.999 INFO:teuthology.orchestra.run.trial098.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.52528.log.gz 2026-02-22T20:15:32.999 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46585.log.gz 2026-02-22T20:15:32.999 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.40218.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42491.log 2026-02-22T20:15:33.000 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.52920.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.40218.log.gz 2026-02-22T20:15:33.000 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47640.log 2026-02-22T20:15:33.000 INFO:teuthology.orchestra.run.trial098.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52920.log.gz 2026-02-22T20:15:33.000 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.42491.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29560.log 2026-02-22T20:15:33.000 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.42491.log.gz 2026-02-22T20:15:33.000 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.47640.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22713.log 2026-02-22T20:15:33.001 INFO:teuthology.orchestra.run.trial098.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.47640.log.gz 2026-02-22T20:15:33.001 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.29560.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29312.log 2026-02-22T20:15:33.001 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29560.log.gz 2026-02-22T20:15:33.001 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.22713.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20650.log 2026-02-22T20:15:33.002 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.29312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44125.log 2026-02-22T20:15:33.002 INFO:teuthology.orchestra.run.trial098.stderr: 90.7% -- replaced with /var/log/ceph/cluster1-client.admin.22713.log.gz 2026-02-22T20:15:33.002 INFO:teuthology.orchestra.run.trial098.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.48275.log.gz 2026-02-22T20:15:33.002 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.29312.log.gz 2026-02-22T20:15:33.002 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.20650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20179.log 2026-02-22T20:15:33.002 INFO:teuthology.orchestra.run.trial098.stderr: 57.2% -- replaced with /var/log/ceph/cluster2-client.admin.20650.log.gz 2026-02-22T20:15:33.002 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.44125.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45224.log 2026-02-22T20:15:33.003 INFO:teuthology.orchestra.run.trial098.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44125.log.gz 2026-02-22T20:15:33.003 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51206.log 2026-02-22T20:15:33.003 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.20179.log: /var/log/ceph/cluster2-client.mirror.45224.log: 58.0% -- replaced with /var/log/ceph/cluster1-client.mirror.20179.log.gz 2026-02-22T20:15:33.003 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30607.log 2026-02-22T20:15:33.003 INFO:teuthology.orchestra.run.trial098.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.45224.log.gz 2026-02-22T20:15:33.003 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.51206.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45298.log 2026-02-22T20:15:33.004 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.51206.log.gz 2026-02-22T20:15:33.004 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.30607.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44517.log 2026-02-22T20:15:33.004 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.30607.log.gz 2026-02-22T20:15:33.004 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45298.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43948.log 2026-02-22T20:15:33.004 INFO:teuthology.orchestra.run.trial098.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.45298.log.gz 2026-02-22T20:15:33.005 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24803.log 2026-02-22T20:15:33.005 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44517.log.gz 2026-02-22T20:15:33.005 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.43948.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38617.log 2026-02-22T20:15:33.005 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.24803.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25727.log 2026-02-22T20:15:33.005 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.24803.log.gz 2026-02-22T20:15:33.006 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.38617.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.43948.log.gz 2026-02-22T20:15:33.006 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58472.log 2026-02-22T20:15:33.006 INFO:teuthology.orchestra.run.trial098.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38617.log.gz 2026-02-22T20:15:33.006 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.25727.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.25130.log 2026-02-22T20:15:33.007 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster2-client.admin.58472.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.34234.log 2026-02-22T20:15:33.007 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.1.25130.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25727.log.gz 2026-02-22T20:15:33.007 INFO:teuthology.orchestra.run.trial098.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.1.25130.log.gz 2026-02-22T20:15:33.007 INFO:teuthology.orchestra.run.trial098.stderr: 87.2%gzip -- replaced with /var/log/ceph/cluster2-client.admin.58472.log.gz -5 2026-02-22T20:15:33.007 INFO:teuthology.orchestra.run.trial098.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.35053.log 2026-02-22T20:15:33.007 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.34234.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20407.log 2026-02-22T20:15:33.007 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.34234.log.gz 2026-02-22T20:15:33.008 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.35053.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36251.log 2026-02-22T20:15:33.008 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.35053.log.gz 2026-02-22T20:15:33.008 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.20407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30817.log 2026-02-22T20:15:33.008 INFO:teuthology.orchestra.run.trial098.stderr: 67.2% -- replaced with /var/log/ceph/cluster2-client.admin.20407.log.gz 2026-02-22T20:15:33.008 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.36251.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32721.log 2026-02-22T20:15:33.009 INFO:teuthology.orchestra.run.trial098.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.36251.log.gz 2026-02-22T20:15:33.009 INFO:teuthology.orchestra.run.trial098.stderr:gzip/var/log/ceph/cluster2-client.admin.30817.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44497.log 2026-02-22T20:15:33.009 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.32721.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.30817.log.gz 2026-02-22T20:15:33.009 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38951.log 2026-02-22T20:15:33.009 INFO:teuthology.orchestra.run.trial098.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.32721.log.gz 2026-02-22T20:15:33.009 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.44497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53332.log 2026-02-22T20:15:33.010 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.38951.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.20708.log 2026-02-22T20:15:33.010 INFO:teuthology.orchestra.run.trial098.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44497.log.gz 2026-02-22T20:15:33.010 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.38951.log.gz 2026-02-22T20:15:33.010 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.53332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46871.log 2026-02-22T20:15:33.011 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.1.20708.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.53332.log.gz 2026-02-22T20:15:33.011 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33139.log 2026-02-22T20:15:33.011 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.mirror.46871.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32874.log 2026-02-22T20:15:33.011 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20196.log 2026-02-22T20:15:33.011 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.33139.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46871.log.gz 2026-02-22T20:15:33.012 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45274.log 2026-02-22T20:15:33.012 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.32874.log: /var/log/ceph/cluster2-client.admin.20196.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.20196.log.gz 2026-02-22T20:15:33.012 INFO:teuthology.orchestra.run.trial098.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.33139.log.gz 2026-02-22T20:15:33.012 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51329.log 2026-02-22T20:15:33.012 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.32874.log.gz 2026-02-22T20:15:33.012 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.45274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25801.log 2026-02-22T20:15:33.013 INFO:teuthology.orchestra.run.trial098.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45274.log.gz 2026-02-22T20:15:33.013 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.mirror.51329.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38101.log 2026-02-22T20:15:33.013 INFO:teuthology.orchestra.run.trial098.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.20708.log.gz 2026-02-22T20:15:33.013 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.25801.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34928.log 2026-02-22T20:15:33.013 INFO:teuthology.orchestra.run.trial098.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.25801.log.gz 2026-02-22T20:15:33.014 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster1-client.admin.38101.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26276.log 2026-02-22T20:15:33.014 INFO:teuthology.orchestra.run.trial098.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51329.log.gz 2026-02-22T20:15:33.014 INFO:teuthology.orchestra.run.trial098.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38101.log.gz 2026-02-22T20:15:33.014 INFO:teuthology.orchestra.run.trial098.stderr:/var/log/ceph/cluster2-client.admin.34928.log: /var/log/ceph/cluster2-client.admin.26276.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.34928.log.gz 2026-02-22T20:15:33.014 INFO:teuthology.orchestra.run.trial098.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.26276.log.gz 2026-02-22T20:15:33.082 INFO:teuthology.orchestra.run.trial012.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-22T20:15:33.196 INFO:teuthology.orchestra.run.trial098.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-22T20:15:37.301 INFO:teuthology.orchestra.run.trial012.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-22T20:15:37.432 INFO:teuthology.orchestra.run.trial012.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-22T20:15:37.502 INFO:teuthology.orchestra.run.trial098.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-22T20:15:37.691 INFO:teuthology.orchestra.run.trial012.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-22T20:15:37.692 INFO:teuthology.orchestra.run.trial012.stderr: 2026-02-22T20:15:37.692 INFO:teuthology.orchestra.run.trial012.stderr:real 0m5.193s 2026-02-22T20:15:37.692 INFO:teuthology.orchestra.run.trial012.stderr:user 0m14.944s 2026-02-22T20:15:37.692 INFO:teuthology.orchestra.run.trial012.stderr:sys 0m0.748s 2026-02-22T20:15:37.797 INFO:teuthology.orchestra.run.trial098.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-22T20:15:38.278 INFO:teuthology.orchestra.run.trial098.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-22T20:15:38.279 INFO:teuthology.orchestra.run.trial098.stderr: 2026-02-22T20:15:38.279 INFO:teuthology.orchestra.run.trial098.stderr:real 0m5.770s 2026-02-22T20:15:38.279 INFO:teuthology.orchestra.run.trial098.stderr:user 0m17.499s 2026-02-22T20:15:38.279 INFO:teuthology.orchestra.run.trial098.stderr:sys 0m1.266s 2026-02-22T20:15:38.281 INFO:tasks.ceph:Archiving logs... 2026-02-22T20:15:38.281 DEBUG:teuthology.misc:Transferring archived files from trial012:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002/remote/trial012/log 2026-02-22T20:15:38.283 DEBUG:teuthology.orchestra.run.trial012:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T20:15:39.426 DEBUG:teuthology.misc:Transferring archived files from trial098:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002/remote/trial098/log 2026-02-22T20:15:39.427 DEBUG:teuthology.orchestra.run.trial098:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T20:15:41.398 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-22T20:15:41.424 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-22T20:15:41.425 DEBUG:teuthology.orchestra.run.trial012:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T20:15:41.428 DEBUG:teuthology.orchestra.run.trial098:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T20:15:41.587 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-22T20:15:41.600 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-22T20:15:41.601 DEBUG:teuthology.orchestra.run.trial012:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T20:15:41.602 DEBUG:teuthology.orchestra.run.trial098:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T20:15:41.665 INFO:teuthology.orchestra.run.trial098.stdout: remote refid st t when poll reach delay offset jitter 2026-02-22T20:15:41.665 INFO:teuthology.orchestra.run.trial098.stdout:============================================================================== 2026-02-22T20:15:41.665 INFO:teuthology.orchestra.run.trial098.stdout:+23.186.168.130 132.163.97.4 2 u 141 64 374 69.660 -3.360 3.366 2026-02-22T20:15:41.665 INFO:teuthology.orchestra.run.trial098.stdout:-172-234-25-10.i 82.43.52.28 2 u 61 64 377 27.279 +4.321 0.497 2026-02-22T20:15:41.665 INFO:teuthology.orchestra.run.trial098.stdout:*69.172.133.130 68.234.48.70 2 u 4 64 377 4.705 -2.864 1.091 2026-02-22T20:15:41.665 INFO:teuthology.orchestra.run.trial098.stdout:+owners.kjsl.com 75.114.196.163 2 u 30 64 377 64.662 -3.267 1.766 2026-02-22T20:15:41.666 INFO:teuthology.orchestra.run.trial012.stdout: remote refid st t when poll reach delay offset jitter 2026-02-22T20:15:41.666 INFO:teuthology.orchestra.run.trial012.stdout:============================================================================== 2026-02-22T20:15:41.666 INFO:teuthology.orchestra.run.trial012.stdout:+23.186.168.130 132.163.97.4 2 u 2 64 377 69.642 +0.412 0.159 2026-02-22T20:15:41.666 INFO:teuthology.orchestra.run.trial012.stdout:-172-234-25-10.i 82.43.52.28 2 u 65 64 377 27.335 +7.727 0.371 2026-02-22T20:15:41.666 INFO:teuthology.orchestra.run.trial012.stdout:*69.172.133.130 68.234.48.70 2 u 7 64 377 4.694 +1.257 2.326 2026-02-22T20:15:41.666 INFO:teuthology.orchestra.run.trial012.stdout:+owners.kjsl.com 75.114.196.163 2 u 65 64 377 64.614 -0.666 0.644 2026-02-22T20:15:41.666 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-22T20:15:41.678 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-22T20:15:41.678 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-22T20:15:41.692 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-22T20:15:41.706 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-22T20:15:41.720 INFO:teuthology.task.internal:Duration was 1907.387091 seconds 2026-02-22T20:15:41.720 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-22T20:15:41.733 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-22T20:15:41.733 DEBUG:teuthology.orchestra.run.trial012:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T20:15:41.735 DEBUG:teuthology.orchestra.run.trial098:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T20:15:41.759 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-22T20:15:41.759 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial012.front.sepia.ceph.com 2026-02-22T20:15:41.759 DEBUG:teuthology.orchestra.run.trial012:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T20:15:41.806 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial098.front.sepia.ceph.com 2026-02-22T20:15:41.806 DEBUG:teuthology.orchestra.run.trial098:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T20:15:41.813 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-22T20:15:41.813 DEBUG:teuthology.orchestra.run.trial012:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T20:15:41.855 DEBUG:teuthology.orchestra.run.trial098:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T20:15:42.077 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-22T20:15:42.077 DEBUG:teuthology.orchestra.run.trial012:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T20:15:42.079 DEBUG:teuthology.orchestra.run.trial098:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T20:15:42.085 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T20:15:42.086 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T20:15:42.086 INFO:teuthology.orchestra.run.trial012.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T20:15:42.086 INFO:teuthology.orchestra.run.trial012.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T20:15:42.087 INFO:teuthology.orchestra.run.trial012.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T20:15:42.087 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T20:15:42.088 INFO:teuthology.orchestra.run.trial098.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T20:15:42.088 INFO:teuthology.orchestra.run.trial098.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T20:15:42.088 INFO:teuthology.orchestra.run.trial098.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T20:15:42.088 INFO:teuthology.orchestra.run.trial098.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T20:15:42.154 INFO:teuthology.orchestra.run.trial012.stderr: 81.4% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T20:15:42.158 INFO:teuthology.orchestra.run.trial098.stderr: 81.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T20:15:42.160 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-22T20:15:42.178 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-22T20:15:42.178 DEBUG:teuthology.orchestra.run.trial012:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T20:15:42.210 DEBUG:teuthology.orchestra.run.trial098:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T20:15:42.219 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-22T20:15:42.236 DEBUG:teuthology.orchestra.run.trial012:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T20:15:42.254 DEBUG:teuthology.orchestra.run.trial098:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T20:15:42.260 INFO:teuthology.orchestra.run.trial012.stdout:kernel.core_pattern = core 2026-02-22T20:15:42.270 INFO:teuthology.orchestra.run.trial098.stdout:kernel.core_pattern = core 2026-02-22T20:15:42.278 DEBUG:teuthology.orchestra.run.trial012:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T20:15:42.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T20:15:42.313 DEBUG:teuthology.orchestra.run.trial098:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T20:15:42.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T20:15:42.327 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-22T20:15:42.344 INFO:teuthology.task.internal:Transferring archived files... 2026-02-22T20:15:42.345 DEBUG:teuthology.misc:Transferring archived files from trial012:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002/remote/trial012 2026-02-22T20:15:42.346 DEBUG:teuthology.orchestra.run.trial012:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T20:15:42.384 DEBUG:teuthology.misc:Transferring archived files from trial098:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64002/remote/trial098 2026-02-22T20:15:42.384 DEBUG:teuthology.orchestra.run.trial098:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T20:15:42.414 INFO:teuthology.task.internal:Removing archive directory... 2026-02-22T20:15:42.414 DEBUG:teuthology.orchestra.run.trial012:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T20:15:42.416 DEBUG:teuthology.orchestra.run.trial098:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T20:15:42.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-22T20:15:42.460 INFO:teuthology.task.internal:Not uploading archives. 2026-02-22T20:15:42.460 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-22T20:15:42.474 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-22T20:15:42.474 DEBUG:teuthology.orchestra.run.trial012:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T20:15:42.476 DEBUG:teuthology.orchestra.run.trial098:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T20:15:42.479 INFO:teuthology.orchestra.run.trial012.stdout: 1179659 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 20:15 /home/ubuntu/cephtest 2026-02-22T20:15:42.486 INFO:teuthology.orchestra.run.trial098.stdout: 1179659 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 20:15 /home/ubuntu/cephtest 2026-02-22T20:15:42.487 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-22T20:15:42.501 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-22T20:15:42.534 INFO:teuthology.run:Summary data: description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zstd supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-workunit-minimum} duration: 1907.3870911598206 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-22T20:15:42.534 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T20:15:42.633 INFO:teuthology.run:pass